513 lines
19 KiB
Python
513 lines
19 KiB
Python
# Copyright 2017 The Armada Authors.
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
import inspect
|
||
import os
|
||
import shutil
|
||
|
||
import fixtures
|
||
from hapi.chart.chart_pb2 import Chart
|
||
from hapi.chart.metadata_pb2 import Metadata
|
||
import mock
|
||
import testtools
|
||
import yaml
|
||
|
||
from armada import const
|
||
from armada.handlers.chartbuilder import ChartBuilder
|
||
from armada.exceptions import chartbuilder_exceptions
|
||
|
||
|
||
class BaseChartBuilderTestCase(testtools.TestCase):
|
||
chart_yaml = """
|
||
apiVersion: v1
|
||
description: A sample Helm chart for Kubernetes
|
||
name: hello-world-chart
|
||
version: 0.1.0
|
||
"""
|
||
|
||
chart_value = """
|
||
# Default values for hello-world-chart.
|
||
# This is a YAML-formatted file.
|
||
# Declare variables to be passed into your templates.
|
||
replicaCount: 1
|
||
image:
|
||
repository: nginx
|
||
tag: stable
|
||
pullPolicy: IfNotPresent
|
||
service:
|
||
name: nginx
|
||
type: ClusterIP
|
||
externalPort: 38443
|
||
internalPort: 80
|
||
resources:
|
||
limits:
|
||
cpu: 100m
|
||
memory: 128Mi
|
||
requests:
|
||
cpu: 100m
|
||
memory: 128Mi
|
||
"""
|
||
|
||
chart_stream = """
|
||
schema: armada/Chart/v1
|
||
metadata:
|
||
name: test
|
||
data:
|
||
chart_name: mariadb
|
||
release: mariadb
|
||
namespace: openstack
|
||
install:
|
||
no_hooks: false
|
||
upgrade:
|
||
no_hooks: false
|
||
values:
|
||
replicas: 1
|
||
volume:
|
||
size: 1Gi
|
||
source:
|
||
type: git
|
||
location: git://opendev.org/openstack/openstack-helm
|
||
subpath: mariadb
|
||
reference: master
|
||
dependencies: []
|
||
"""
|
||
|
||
dependency_chart_yaml = """
|
||
apiVersion: v1
|
||
description: Another sample Helm chart for Kubernetes
|
||
name: dependency-chart
|
||
version: 0.1.0
|
||
"""
|
||
|
||
dependency_chart_stream = """
|
||
schema: armada/Chart/v1
|
||
metadata:
|
||
name: dep
|
||
data:
|
||
chart_name: keystone
|
||
release: keystone
|
||
namespace: undercloud
|
||
timeout: 100
|
||
install:
|
||
no_hooks: false
|
||
upgrade:
|
||
no_hooks: false
|
||
values: {}
|
||
source:
|
||
type: git
|
||
location: git://github.com/example/example
|
||
subpath: example-chart
|
||
reference: master
|
||
dependencies: []
|
||
"""
|
||
|
||
def _write_temporary_file_contents(self, directory, filename, contents):
|
||
path = os.path.join(directory, filename)
|
||
fd = os.open(path, os.O_CREAT | os.O_WRONLY)
|
||
try:
|
||
os.write(fd, contents.encode('utf-8'))
|
||
finally:
|
||
os.close(fd)
|
||
|
||
def _make_temporary_subdirectory(self, parent_path, sub_path):
|
||
subdir = os.path.join(parent_path, sub_path)
|
||
if not os.path.exists(subdir):
|
||
os.makedirs(subdir)
|
||
self.addCleanup(shutil.rmtree, subdir)
|
||
return subdir
|
||
|
||
def _get_test_chart(self, chart_dir):
|
||
return {
|
||
'schema': 'armada/Chart/v1',
|
||
'metadata': {
|
||
'name': 'test'
|
||
},
|
||
const.KEYWORD_DATA: {
|
||
'source_dir': (chart_dir.path, '')
|
||
}
|
||
}
|
||
|
||
|
||
class ChartBuilderTestCase(BaseChartBuilderTestCase):
|
||
def test_source_clone(self):
|
||
# Create a temporary directory with Chart.yaml that contains data
|
||
# from ``self.chart_yaml``.
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'Chart.yaml', self.chart_yaml)
|
||
|
||
chartbuilder = ChartBuilder.from_chart_doc(
|
||
self._get_test_chart(chart_dir))
|
||
|
||
# Validate response type is :class:`hapi.chart.metadata_pb2.Metadata`
|
||
resp = chartbuilder.get_metadata()
|
||
self.assertIsInstance(resp, Metadata)
|
||
|
||
def test_get_metadata_with_incorrect_file_invalid(self):
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
|
||
chartbuilder = ChartBuilder.from_chart_doc(
|
||
self._get_test_chart(chart_dir))
|
||
|
||
self.assertRaises(
|
||
chartbuilder_exceptions.MetadataLoadException,
|
||
chartbuilder.get_metadata)
|
||
|
||
def test_get_files(self):
|
||
"""Validates that ``get_files()`` ignores 'Chart.yaml', 'values.yaml'
|
||
and 'templates' subfolder and all the files contained therein.
|
||
"""
|
||
|
||
# Create a temporary directory that represents a chart source directory
|
||
# with various files, including 'Chart.yaml' and 'values.yaml' which
|
||
# should be ignored by `get_files()`.
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']:
|
||
self._write_temporary_file_contents(chart_dir.path, filename, "")
|
||
|
||
# Create a template directory -- 'templates' -- nested inside the chart
|
||
# directory which should also be ignored.
|
||
templates_subdir = self._make_temporary_subdirectory(
|
||
chart_dir.path, 'templates')
|
||
for filename in ['template%d' % x for x in range(3)]:
|
||
self._write_temporary_file_contents(templates_subdir, filename, "")
|
||
|
||
chartbuilder = ChartBuilder.from_chart_doc(
|
||
self._get_test_chart(chart_dir))
|
||
|
||
expected_files = (
|
||
'[type_url: "%s"\n, type_url: "%s"\n]' % ('./bar', './foo'))
|
||
# Validate that only 'foo' and 'bar' are returned.
|
||
actual_files = sorted(
|
||
chartbuilder.get_files(), key=lambda x: x.type_url)
|
||
self.assertEqual(expected_files, repr(actual_files).strip())
|
||
|
||
def test_get_files_with_unicode_characters(self):
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']:
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1")
|
||
|
||
chartbuilder = ChartBuilder.from_chart_doc(
|
||
self._get_test_chart(chart_dir))
|
||
chartbuilder.get_files()
|
||
|
||
def test_get_basic_helm_chart(self):
|
||
# Before ChartBuilder is executed the `source_dir` points to a
|
||
# directory that was either clone or unpacked from a tarball... pretend
|
||
# that that logic has already been performed.
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'Chart.yaml', self.chart_yaml)
|
||
ch = yaml.safe_load(self.chart_stream)
|
||
ch['data']['source_dir'] = (chart_dir.path, '')
|
||
|
||
test_chart = ch
|
||
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
|
||
helm_chart = chartbuilder.get_helm_chart()
|
||
|
||
expected = inspect.cleandoc(
|
||
"""
|
||
metadata {
|
||
name: "hello-world-chart"
|
||
version: "0.1.0"
|
||
description: "A sample Helm chart for Kubernetes"
|
||
}
|
||
values {
|
||
}
|
||
""").strip()
|
||
|
||
self.assertIsInstance(helm_chart, Chart)
|
||
self.assertTrue(hasattr(helm_chart, 'metadata'))
|
||
self.assertTrue(hasattr(helm_chart, 'values'))
|
||
self.assertEqual(expected, repr(helm_chart).strip())
|
||
|
||
def test_get_helm_chart_with_values(self):
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'Chart.yaml', self.chart_yaml)
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'values.yaml', self.chart_value)
|
||
|
||
ch = yaml.safe_load(self.chart_stream)
|
||
ch['data']['source_dir'] = (chart_dir.path, '')
|
||
|
||
test_chart = ch
|
||
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
|
||
helm_chart = chartbuilder.get_helm_chart()
|
||
|
||
self.assertIsInstance(helm_chart, Chart)
|
||
self.assertTrue(hasattr(helm_chart, 'metadata'))
|
||
self.assertTrue(hasattr(helm_chart, 'values'))
|
||
self.assertTrue(hasattr(helm_chart.values, 'raw'))
|
||
self.assertEqual(self.chart_value, helm_chart.values.raw)
|
||
|
||
def test_get_helm_chart_with_files(self):
|
||
# Create a chart directory with some test files.
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
# Chart.yaml is mandatory for `ChartBuilder.get_metadata`.
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'Chart.yaml', self.chart_yaml)
|
||
self._write_temporary_file_contents(chart_dir.path, 'foo', "foobar")
|
||
self._write_temporary_file_contents(chart_dir.path, 'bar', "bazqux")
|
||
|
||
# Also create a nested directory and verify that files from it are also
|
||
# added.
|
||
nested_dir = self._make_temporary_subdirectory(
|
||
chart_dir.path, 'nested')
|
||
self._write_temporary_file_contents(nested_dir, 'nested0', "random")
|
||
|
||
ch = yaml.safe_load(self.chart_stream)
|
||
ch['data']['source_dir'] = (chart_dir.path, '')
|
||
|
||
test_chart = ch
|
||
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
|
||
helm_chart = chartbuilder.get_helm_chart()
|
||
|
||
expected_files = (
|
||
'[type_url: "%s"\nvalue: "bazqux"\n, '
|
||
'type_url: "%s"\nvalue: "foobar"\n, '
|
||
'type_url: "%s"\nvalue: "random"\n]' %
|
||
('./bar', './foo', 'nested/nested0'))
|
||
|
||
self.assertIsInstance(helm_chart, Chart)
|
||
self.assertTrue(hasattr(helm_chart, 'metadata'))
|
||
self.assertTrue(hasattr(helm_chart, 'values'))
|
||
self.assertTrue(hasattr(helm_chart, 'files'))
|
||
actual_files = sorted(helm_chart.files, key=lambda x: x.value)
|
||
self.assertEqual(expected_files, repr(actual_files).strip())
|
||
|
||
def test_get_helm_chart_includes_only_relevant_files(self):
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
|
||
templates_subdir = self._make_temporary_subdirectory(
|
||
chart_dir.path, 'templates')
|
||
charts_subdir = self._make_temporary_subdirectory(
|
||
chart_dir.path, 'charts')
|
||
templates_nested_subdir = self._make_temporary_subdirectory(
|
||
templates_subdir, 'bin')
|
||
charts_nested_subdir = self._make_temporary_subdirectory(
|
||
charts_subdir, 'extra')
|
||
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'Chart.yaml', self.chart_yaml)
|
||
self._write_temporary_file_contents(chart_dir.path, 'foo', "foobar")
|
||
self._write_temporary_file_contents(chart_dir.path, 'bar', "bazqux")
|
||
|
||
# Files to ignore within top-level directory.
|
||
files_to_ignore = ['Chart.yaml', 'values.yaml', 'values.toml']
|
||
for file in files_to_ignore:
|
||
self._write_temporary_file_contents(chart_dir.path, file, "")
|
||
file_to_ignore = 'file_to_ignore'
|
||
# Files to ignore within templates/ subdirectory.
|
||
self._write_temporary_file_contents(
|
||
templates_subdir, file_to_ignore, "")
|
||
# Files to ignore within templates/bin subdirectory.
|
||
self._write_temporary_file_contents(
|
||
templates_nested_subdir, file_to_ignore, "")
|
||
# Files to ignore within charts/extra subdirectory.
|
||
self._write_temporary_file_contents(
|
||
charts_nested_subdir, file_to_ignore, "")
|
||
self._write_temporary_file_contents(
|
||
charts_nested_subdir, 'Chart.yaml', self.chart_yaml)
|
||
# Files to **include** within charts/ subdirectory.
|
||
self._write_temporary_file_contents(charts_subdir, '.prov', "xyzzy")
|
||
|
||
ch = yaml.safe_load(self.chart_stream)
|
||
ch['data']['source_dir'] = (chart_dir.path, '')
|
||
|
||
test_chart = ch
|
||
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
|
||
helm_chart = chartbuilder.get_helm_chart()
|
||
|
||
expected_files = (
|
||
'[type_url: "%s"\nvalue: "bazqux"\n, '
|
||
'type_url: "%s"\nvalue: "foobar"\n, '
|
||
'type_url: "%s"\nvalue: "xyzzy"\n]' %
|
||
('./bar', './foo', 'charts/.prov'))
|
||
|
||
# Validate that only relevant files are included, that the ignored
|
||
# files are present.
|
||
self.assertIsInstance(helm_chart, Chart)
|
||
self.assertTrue(hasattr(helm_chart, 'metadata'))
|
||
self.assertTrue(hasattr(helm_chart, 'values'))
|
||
self.assertTrue(hasattr(helm_chart, 'files'))
|
||
actual_files = sorted(helm_chart.files, key=lambda x: x.value)
|
||
self.assertEqual(expected_files, repr(actual_files).strip())
|
||
|
||
def test_get_helm_chart_with_dependencies(self):
|
||
# Main chart directory and files.
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'Chart.yaml', self.chart_yaml)
|
||
ch = yaml.safe_load(self.chart_stream)
|
||
ch['data']['source_dir'] = (chart_dir.path, '')
|
||
|
||
# Dependency chart directory and files.
|
||
dep_chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, dep_chart_dir.path)
|
||
self._write_temporary_file_contents(
|
||
dep_chart_dir.path, 'Chart.yaml', self.dependency_chart_yaml)
|
||
dep_ch = yaml.safe_load(self.dependency_chart_stream)
|
||
dep_ch['data']['source_dir'] = (dep_chart_dir.path, '')
|
||
|
||
main_chart = ch
|
||
dependency_chart = dep_ch
|
||
main_chart['data']['dependencies'] = [dependency_chart]
|
||
|
||
chartbuilder = ChartBuilder.from_chart_doc(main_chart)
|
||
helm_chart = chartbuilder.get_helm_chart()
|
||
|
||
expected_dependency = inspect.cleandoc(
|
||
"""
|
||
metadata {
|
||
name: "dependency-chart"
|
||
version: "0.1.0"
|
||
description: "Another sample Helm chart for Kubernetes"
|
||
}
|
||
values {
|
||
}
|
||
""").strip()
|
||
|
||
expected = inspect.cleandoc(
|
||
"""
|
||
metadata {
|
||
name: "hello-world-chart"
|
||
version: "0.1.0"
|
||
description: "A sample Helm chart for Kubernetes"
|
||
}
|
||
dependencies {
|
||
metadata {
|
||
name: "dependency-chart"
|
||
version: "0.1.0"
|
||
description: "Another sample Helm chart for Kubernetes"
|
||
}
|
||
values {
|
||
}
|
||
}
|
||
values {
|
||
}
|
||
""").strip()
|
||
|
||
# Validate the main chart.
|
||
self.assertIsInstance(helm_chart, Chart)
|
||
self.assertTrue(hasattr(helm_chart, 'metadata'))
|
||
self.assertTrue(hasattr(helm_chart, 'values'))
|
||
self.assertEqual(expected, repr(helm_chart).strip())
|
||
|
||
# Validate the dependency chart.
|
||
self.assertTrue(hasattr(helm_chart, 'dependencies'))
|
||
self.assertEqual(1, len(helm_chart.dependencies))
|
||
|
||
dep_helm_chart = helm_chart.dependencies[0]
|
||
self.assertIsInstance(dep_helm_chart, Chart)
|
||
self.assertTrue(hasattr(dep_helm_chart, 'metadata'))
|
||
self.assertTrue(hasattr(dep_helm_chart, 'values'))
|
||
self.assertEqual(expected_dependency, repr(dep_helm_chart).strip())
|
||
|
||
def test_dump(self):
|
||
# Validate base case.
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, 'Chart.yaml', self.chart_yaml)
|
||
ch = yaml.safe_load(self.chart_stream)
|
||
ch['data']['source_dir'] = (chart_dir.path, '')
|
||
|
||
test_chart = ch
|
||
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
|
||
self.assertRegex(
|
||
repr(chartbuilder.dump()),
|
||
'hello-world-chart.*A sample Helm chart for Kubernetes.*')
|
||
|
||
# Validate recursive case (with dependencies).
|
||
dep_chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, dep_chart_dir.path)
|
||
self._write_temporary_file_contents(
|
||
dep_chart_dir.path, 'Chart.yaml', self.dependency_chart_yaml)
|
||
dep_ch = yaml.safe_load(self.dependency_chart_stream)
|
||
dep_ch['data']['source_dir'] = (dep_chart_dir.path, '')
|
||
|
||
dependency_chart = dep_ch
|
||
test_chart['data']['dependencies'] = [dependency_chart]
|
||
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
|
||
|
||
re = inspect.cleandoc(
|
||
"""
|
||
hello-world-chart.*A sample Helm chart for Kubernetes.*
|
||
dependency-chart.*Another sample Helm chart for Kubernetes.*
|
||
""").replace('\n', '').strip()
|
||
self.assertRegex(repr(chartbuilder.dump()), re)
|
||
|
||
|
||
class ChartBuilderNegativeTestCase(BaseChartBuilderTestCase):
|
||
def setUp(self):
|
||
super(ChartBuilderNegativeTestCase, self).setUp()
|
||
# Create an exception for testing since instantiating one manually
|
||
# is tedious.
|
||
try:
|
||
str(b'\xff', 'utf8')
|
||
except UnicodeDecodeError as e:
|
||
self.exc_to_raise = e
|
||
else:
|
||
self.fail('Failed to create an exception needed for testing.')
|
||
|
||
def test_get_files_always_fails_to_read_binary_file_raises_exc(self):
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']:
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1")
|
||
|
||
chartbuilder = ChartBuilder.from_chart_doc(
|
||
self._get_test_chart(chart_dir))
|
||
|
||
# Confirm it failed for both encodings.
|
||
error_re = (
|
||
r'.*A str exception occurred while trying to read file:'
|
||
r'.*Details:\n.*\(encoding=utf-8\).*\n\(encoding=latin1\)')
|
||
with mock.patch("builtins.open", mock.mock_open(read_data="")) \
|
||
as mock_file:
|
||
mock_file.return_value.read.side_effect = self.exc_to_raise
|
||
self.assertRaisesRegexp(
|
||
chartbuilder_exceptions.FilesLoadException, error_re,
|
||
chartbuilder.get_files)
|
||
|
||
def test_get_files_fails_once_to_read_binary_file_passes(self):
|
||
chart_dir = self.useFixture(fixtures.TempDir())
|
||
self.addCleanup(shutil.rmtree, chart_dir.path)
|
||
files = ['foo', 'bar']
|
||
for filename in files:
|
||
self._write_temporary_file_contents(
|
||
chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1")
|
||
|
||
chartbuilder = ChartBuilder.from_chart_doc(
|
||
self._get_test_chart(chart_dir))
|
||
|
||
side_effects = [self.exc_to_raise, "", ""]
|
||
with mock.patch("builtins.open", mock.mock_open(read_data="")) \
|
||
as mock_file:
|
||
mock_file.return_value.read.side_effect = side_effects
|
||
chartbuilder.get_files()
|