feat(manifest): ability to override manifest value

- Add --set flag to override manifest values from CLI
- Add --values flag to override manifest values from values file
- Add support to override manifests values with API values option

Closes #146

Change-Id: Iefa14e4d3005aab3ee803ffb65dfe1a867507c0e
This commit is contained in:
drewwalters96 2017-08-09 08:33:29 -05:00 committed by gardlt
parent 767cac7d57
commit be8e86351f
16 changed files with 593 additions and 19 deletions

View File

@ -24,10 +24,12 @@ def applyCharts(args):
args.disable_update_post,
args.enable_chart_cleanup,
args.dry_run,
args.set,
args.wait,
args.timeout,
args.tiller_host,
args.tiller_port,
args.values,
args.debug_logging)
armada.sync()
@ -47,6 +49,8 @@ class ApplyChartsCommand(cmd.Command):
default=False, help='Disable post upgrade actions')
parser.add_argument('--enable-chart-cleanup', action='store_true',
default=False, help='Enable Chart Clean Up')
parser.add_argument('--set', action='append', help='Override Armada'
' manifest values.')
parser.add_argument('--wait', action='store_true',
default=False, help='Wait until all charts'
'have been deployed')
@ -55,8 +59,13 @@ class ApplyChartsCommand(cmd.Command):
' for charts to deploy')
parser.add_argument('--tiller-host', action='store', type=str,
help='Specify the tiller host')
parser.add_argument('--tiller-port', action='store', type=int,
default=44134, help='Specify the tiller port')
parser.add_argument('--values', action='append',
help='Override manifest values with a yaml file')
return parser
def take_action(self, parsed_args):

View File

@ -23,6 +23,7 @@ KEYWORD_PREFIX = 'release_prefix'
KEYWORD_GROUPS = 'chart_groups'
KEYWORD_CHARTS = 'chart_group'
KEYWORD_RELEASE = 'release'
KEYWORD_CHART = 'chart'
# Statuses
STATUS_DEPLOYED = 'DEPLOYED'

View File

@ -0,0 +1,70 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from armada.exceptions import base_exception
class OverrideException(base_exception.ArmadaBaseException):
'''
Base class for Override handler exception and error handling.
'''
message = 'An unknown Override handler error occured.'
class InvalidOverrideTypeException(OverrideException):
'''
Exception that occurs when an invalid override type is used with the
set flag.
'''
def __init__(self, override_type):
self._message = 'Override type "{}" is invalid'.format(override_type)
super(InvalidOverrideTypeException, self).__init__(self._message)
class InvalidOverrideFileException(OverrideException):
'''
Exception that occurs when an invalid override file is provided.
'''
def __init__(self, filename):
self._message = '{} is not a valid override file.'.format(filename)
super(InvalidOverrideFileException, self).__init__(self._message)
class InvalidOverrideValueException(OverrideException):
'''
Exception that occurs when an invalid value is used with the set flag.
'''
def __init__(self, override_command):
self._message = '{} is not a valid override statement.'.format(
override_command)
super(InvalidOverrideValueException, self).__init__(self._message)
class UnknownDocumentOverrideException(OverrideException):
'''
Exception that occurs when an invalid value is used with the set flag.
'''
def __init__(self, doc_type, doc_name):
self._message = 'Unable to find {1} document schema: {0} '.format(
doc_type, doc_name)
super(UnknownDocumentOverrideException, self).__init__(self._message)

View File

@ -19,8 +19,9 @@ from oslo_log import log as logging
from supermutes.dot import dotify
from armada.handlers.chartbuilder import ChartBuilder
from armada.handlers.tiller import Tiller
from armada.handlers.manifest import Manifest
from armada.handlers.override import Override
from armada.handlers.tiller import Tiller
from armada.exceptions import armada_exceptions
from armada.exceptions import source_exceptions
from armada.exceptions import lint_exceptions
@ -47,10 +48,12 @@ class Armada(object):
disable_update_post=False,
enable_chart_cleanup=False,
dry_run=False,
set_ovr=None,
wait=False,
timeout=DEFAULT_TIMEOUT,
tiller_host=None,
tiller_port=44134,
values=None,
debug=False):
'''
Initialize the Armada Engine and establish
@ -61,9 +64,11 @@ class Armada(object):
self.disable_update_post = disable_update_post
self.enable_chart_cleanup = enable_chart_cleanup
self.dry_run = dry_run
self.overrides = set_ovr
self.wait = wait
self.timeout = timeout
self.tiller = Tiller(tiller_host=tiller_host, tiller_port=tiller_port)
self.values = values
self.documents = list(yaml.safe_load_all(file))
self.config = None
self.debug = debug
@ -89,18 +94,25 @@ class Armada(object):
Perform a series of checks and operations to ensure proper deployment
'''
# Ensure tiller is available and yaml is valid
# Ensure tiller is available and manifest is valid
if not self.tiller.tiller_status():
raise tiller_exceptions.TillerServicesUnavailableException()
if not lint.validate_armada_documents(self.documents):
raise lint_exceptions.InvalidManifestException()
# Override manifest values if --set flag is used
if self.overrides or self.values:
self.documents = Override(
self.documents, overrides=self.overrides,
values=self.values).update_manifests()
# Get config and validate
self.config = self.get_armada_manifest()
if not lint.validate_armada_object(self.config):
raise lint_exceptions.InvalidArmadaObjectExceptionl()
raise lint_exceptions.InvalidArmadaObjectException()
self.config = self.get_armada_manifest()
# Purge known releases that have failed and are in the current yaml
prefix = self.config.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_PREFIX)
@ -219,8 +231,6 @@ class Armada(object):
pre_actions = {}
post_actions = {}
LOG.info('%s', chart.release)
if chart.release is None:
continue
@ -231,8 +241,8 @@ class Armada(object):
chart_timeout = self.timeout
if chart_wait:
if chart_timeout == DEFAULT_TIMEOUT:
chart_timeout = getattr(chart, 'timeout',
chart_timeout)
chart_timeout = getattr(
chart, 'timeout', DEFAULT_TIMEOUT)
chartbuilder = ChartBuilder(chart)
protoc_chart = chartbuilder.get_helm_chart()

View File

@ -99,8 +99,12 @@ class Manifest(object):
if isinstance(group, dict):
continue
chart_grp = self.find_chart_group_document(group)
self.manifest['data']['chart_groups'][iter] = chart_grp.get(
'data')
# Add name to chart group
ch_grp_data = chart_grp.get('data')
ch_grp_data['name'] = chart_grp.get('metadata').get('name')
self.manifest['data']['chart_groups'][iter] = ch_grp_data
except Exception:
raise Exception(
"Could not find chart group {} in {}".format(

158
armada/handlers/override.py Normal file
View File

@ -0,0 +1,158 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import json
import yaml
from armada import const
from armada.exceptions import override_exceptions
from armada.utils import lint
class Override(object):
def __init__(self, documents, overrides=None, values=None):
self.documents = documents
self.overrides = overrides
self.values = values
def _load_yaml_file(self, doc):
'''
Retrieve yaml file as a dictionary.
'''
try:
with open(doc) as f:
return list(yaml.safe_load_all(f.read()))
except IOError:
raise override_exceptions.InvalidOverrideFileException(doc)
def update(self, d, u):
for k, v in u.items():
if isinstance(v, collections.Mapping):
r = self.update(d.get(k, {}), v)
d[k] = r
else:
d[k] = u[k]
return d
def find_document_type(self, alias):
if alias == 'chart_group':
return const.DOCUMENT_GROUP
if alias == 'chart':
return const.DOCUMENT_CHART
if alias == 'manifest':
return const.DOCUMENT_MANIFEST
else:
raise ValueError("Could not find {} document".format(alias))
def find_manifest_document(self, doc_path):
for doc in self.documents:
if doc.get('schema') == self.find_document_type(
doc_path[0]) and doc.get('metadata').get(
'name') == doc_path[1]:
return doc
raise override_exceptions.UnknownDocumentOverrideException(
doc_path[0], doc_path[1])
def array_to_dict(self, data_path, new_value):
def convert(data):
if isinstance(data, str):
return str(data)
elif isinstance(data, collections.Mapping):
return dict(map(convert, data.items()))
elif isinstance(data, collections.Iterable):
return type(data)(map(convert, data))
else:
return data
if not new_value:
return
if not data_path:
return
tree = {}
t = tree
for part in data_path:
if part == data_path[-1]:
t.setdefault(part, None)
continue
t = t.setdefault(part, {})
string = json.dumps(tree).replace('null', '"{}"'.format(new_value))
data_obj = convert(json.loads(string, encoding='utf-8'))
return data_obj
def override_manifest_value(self, doc_path, data_path, new_value):
document = self.find_manifest_document(doc_path)
new_data = self.array_to_dict(data_path, new_value)
self.update(document.get('data'), new_data)
def update_document(self, merging_values):
for doc in merging_values:
if doc.get('schema') == const.DOCUMENT_CHART:
self.update_chart_document(doc)
if doc.get('schema') == const.DOCUMENT_GROUP:
self.update_chart_group_document(doc)
if doc.get('schema') == const.DOCUMENT_MANIFEST:
self.update_armada_manifest(doc)
def update_chart_document(self, ovr):
for doc in self.documents:
if doc.get('schema') == const.DOCUMENT_CHART and doc.get(
'metadata').get('name') == ovr.get('metadata').get('name'):
self.update(doc.get('data'), ovr.get('data'))
return
def update_chart_group_document(self, ovr):
for doc in self.documents:
if doc.get('schema') == const.DOCUMENT_GROUP and doc.get(
'metadata').get('name') == ovr.get('metadata').get('name'):
self.update(doc.get('data'), ovr.get('data'))
return
def update_armada_manifest(self, ovr):
for doc in self.documents:
if doc.get('schema') == const.DOCUMENT_MANIFEST and doc.get(
'metadata').get('name') == ovr.get('metadata').get('name'):
self.update(doc.get('data'), ovr.get('data'))
return
def update_manifests(self):
if self.values:
for value in self.values:
merging_values = self._load_yaml_file(value)
self.update_document(merging_values)
if self.overrides:
for override in self.overrides:
new_value = override.split('=')[1]
doc_path = override.split('=')[0].split(":")
data_path = doc_path.pop().split('.')
self.override_manifest_value(doc_path, data_path, new_value)
try:
lint.validate_armada_documents(self.documents)
except Exception:
raise override_exceptions.InvalidOverrideValueException(
self.overrides)
return self.documents

View File

@ -0,0 +1,35 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/hello-world-chart
subpath: .
reference: master
dependencies: []
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group
data:
description: Deploys Simple Service
sequenced: False
chart_group:
- blog-1
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: simple-armada
data:
release_prefix: armada
chart_groups:
- blog-group

View File

@ -0,0 +1,16 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: blog-blog
values: {}
source:
type: dev
location: https://github.com/namespace/hello-world-chart
subpath: .
reference: master
dependencies: []

View File

@ -0,0 +1,12 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: blog-blog
values: {}
source:
type: dev

View File

@ -0,0 +1,16 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/hello-world-chart
subpath: .
reference: master
dependencies: []

View File

@ -0,0 +1,93 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import yaml
import os
from armada.handlers.override import Override
from armada import const
class OverrideTestCase(unittest.TestCase):
def setUp(self):
self.basepath = os.path.join(os.path.dirname(__file__))
self.base_manifest = '{}/templates/base.yaml'.format(self.basepath)
def test_find_document_type_valid(self):
with open(self.base_manifest) as f:
doc_obj = list(yaml.safe_load_all(f.read()))
ovr = Override(doc_obj)
test_group = ovr.find_document_type('chart_group')
self.assertEqual(test_group, const.DOCUMENT_GROUP)
test_chart = ovr.find_document_type('chart')
self.assertEqual(test_chart, const.DOCUMENT_CHART)
test_manifest = ovr.find_document_type('manifest')
self.assertEqual(test_manifest, const.DOCUMENT_MANIFEST)
def test_find_document_type_invalid(self):
with self.assertRaises(Exception):
with open(self.base_manifest) as f:
doc_obj = list(yaml.safe_load_all(f.read()))
ovr = Override(doc_obj)
ovr.find_document_type('charts')
def test_update_dictionary_valid(self):
expected = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '01')
merge = "{}/templates/override-{}.yaml".format(self.basepath, '01')
with open(self.base_manifest) as f, open(expected) as e, open(
merge) as m:
merging_values = list(yaml.safe_load_all(m.read()))
doc_obj = list(yaml.safe_load_all(f.read()))
doc_path = ['chart', 'blog-1']
ovr = Override(doc_obj)
ovr.update_document(merging_values)
ovr_doc = ovr.find_manifest_document(doc_path)
expect_doc = list(yaml.load_all(e.read()))[0]
self.assertEqual(ovr_doc, expect_doc)
def test_find_manifest_document_valid(self):
expected = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '02')
with open(self.base_manifest) as f, open(expected) as e:
doc_path = ['chart', 'blog-1']
doc_obj = list(yaml.safe_load_all(f.read()))
ovr = Override(doc_obj).find_manifest_document(doc_path)
expected_doc = list(yaml.safe_load_all(e.read()))[0]
self.assertEqual(ovr, expected_doc)
def test_convert_array_to_dict_valid(self):
data_path = ['a', 'b', 'c']
new_value = "dev"
expected_dict = {'a': {'b': {'c': 'dev'}}}
ovr = Override(self.base_manifest).array_to_dict(data_path, new_value)
self.assertEqual(ovr, expected_dict)
def test_convert_array_to_dict_invalid(self):
data_path = ['a', 'b', 'c']
new_value = ""
ovr = Override(self.base_manifest).array_to_dict(data_path, new_value)
self.assertIsNone(ovr)
ovr = Override(self.base_manifest).array_to_dict([], new_value)
self.assertIsNone(ovr)

View File

@ -31,7 +31,7 @@ def validate_armada_object(object):
armada_object = object.get('armada')
if not isinstance(armada_object.get(KEYWORD_PREFIX), str):
if armada_object.get(KEYWORD_PREFIX, None) is None:
raise Exception("Could not find {} keyword".format(KEYWORD_PREFIX))
if not isinstance(armada_object.get(KEYWORD_GROUPS), list):
@ -41,9 +41,9 @@ def validate_armada_object(object):
for group in armada_object.get(KEYWORD_GROUPS):
for chart in group.get(KEYWORD_CHARTS):
chart_obj = chart.get('chart')
if not isinstance(chart_obj.get(KEYWORD_RELEASE), str):
if chart_obj.get(KEYWORD_RELEASE, None) is None:
raise Exception('Could not find {} in {}'.format(
KEYWORD_RELEASE, chart_obj.get('name')))
KEYWORD_RELEASE, chart_obj.get('release')))
return True

View File

@ -14,6 +14,7 @@ Armada Endpoints
:>json boolean disable_update_post
:>json boolean enable_chart_cleanup
:>json boolean skip_pre_flight
:>json object values Override manifest values
:>json boolean dry_run
:>json boolean wait
:>json float timeout
@ -37,7 +38,6 @@ Armada Endpoints
}
}
::
Results:

View File

@ -33,8 +33,7 @@ Usage
Build:
git clone https://github.com/att-comdev/armada
cd armada/
git clone https://github.com/att-comdev/armada && cd armada/
docker build . -t quay.io/attcomdev/armada:latest
2. Run Armada docker container
@ -52,7 +51,8 @@ Usage
.. code:: bash
docker run -d --net host -p 8000:8000 --name armada -v ~/.kube/config:/root/.kube/config -v $(pwd)/examples/:/examples quay.io/attcomdev/armada:latest
docker run -d --net host -p 8000:8000 --name armada -v $(pwd)/etc/:/etc/ -v ~/.kube/:/armada/.kube/ -v $(pwd)/examples/:/examples quay.io/attcomdev/armada:latest
docker exec armada armada --help
3. Check that tiller is Available
@ -64,17 +64,139 @@ Usage
.. code:: bash
docker exec armada armada apply /examples/openstack-helm.yaml [ --debug-logging ]
docker exec armada armada apply /examples/openstack-helm.yaml [ --debug ]
5. Upgrading charts: modify the armada yaml or chart source code and run ``armada
apply`` above
.. code:: bash
docker exec armada armada apply /examples/openstack-helm.yaml [ --debug-logging ]
docker exec armada armada apply /examples/openstack-helm.yaml [ --debug ]
6. To check deployed releases:
.. code:: bash
docker exec armada armada tiller --releases
7. Testing Releases:
.. code:: bash
docker exec armada armada test --release=armada-keystone
OR
docker exec armada armada test --file=/examples/openstack-helm.yaml
Overriding Manifest Values
--------------------------
It is possible to override manifest values from the command line using the
--set and --values flags. When using the set flag, the document type should be
specified first, with the target values following in this manner:
.. code:: bash
armada apply --set [ document_type ]:[ document_name ]:[ data_value ]=[ value ]
Example:
armada apply --set chart:blog-1:release="new-blog"
armada apply --set chart:blog-1:values.blog.new="welcome"
.. note::
When overriding values using the set flag, new values will be inserted if
they do not exist. An error will only occur if the correct pattern is
not used.
There are three types of override types that can be specified:
- chart
- chart_group
- manifest
An example of overriding the location of a chart:
.. code:: bash
armada apply --set chart:[ chart_name ]:source.location=test [ FILE ]
Example:
armada apply --set chart:blog-1:release=test [ FILE ]
An example of overriding the description of a chart group:
.. code:: bash
armada apply --set chart_group:[ chart_group_name ]:description=test [ FILE ]
Example:
armada apply examples/simple.yaml --set chart_group:blog-group:description=test
An example of overriding the release prefix of a manifest:
.. code:: bash
armada apply --set manifest:[ manifest_name ]:release_prefix=[ value ] [ FILE ]
Example:
armada apply example/simple.yaml --set manifest:simple-armada:release_prefix=armada-2
.. note::
The --set flag can be used multiple times.
It is also possible to override manifest values using values specified in a
yaml file using the --values flag. When using the --values flag, a path to the
yaml file should be specified in this format:
.. code:: bash
armada apply --values [ path_to_yaml ] [ FILE ]
Example:
armada apply examples/simple.yaml --values examples/simple-ovr-values.yaml
.. note::
The --values flag, like the --set flag, can be specified more than once.
The --set and --values flag can also be specified at the same time;
however, overrides specified by the --set flag take precedence over those
specified by the --values flag.
When creating a yaml file of override values, it should be the same as creating
an armada manifest overriding documents with the same schema and metadata name
for example:
.. code:: yaml
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
release: chart-example
namespace: blog-blog
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-2
data:
release: chart-example-2
namespace: blog-blog
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group
data:
description: Change value deploy
chart_group:
- blog-1

View File

@ -0,0 +1,22 @@
---
schema: armada/Chart/v1
metadata:
name: blog-1
data:
release: chart-example
namespace: blog-blog
---
schema: armada/Chart/v1
metadata:
name: blog-2
data:
release: chart-example-2
namespace: blog-blog
---
schema: armada/ChartGroup/v1
metadata:
name: blog-group
data:
description: Change value deploy
chart_group:
- blog-1

View File

@ -8,6 +8,10 @@ data:
chart_name: blog-1
release: blog-1
namespace: default
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
@ -26,6 +30,8 @@ data:
namespace: default
values:
some: value
install:
no_hooks: false
upgrade:
no_hooks: false
source: