# Copyright 2017 The Armada Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re from google.protobuf.any_pb2 import Any from hapi.chart.chart_pb2 import Chart from hapi.chart.config_pb2 import Config from hapi.chart.metadata_pb2 import Metadata from hapi.chart.template_pb2 import Template from oslo_config import cfg from oslo_log import log as logging import yaml from armada import const from armada.exceptions import chartbuilder_exceptions from armada.handlers.schema import get_schema_info LOG = logging.getLogger(__name__) CONF = cfg.CONF class ChartBuilder(object): ''' This class handles taking chart intentions as a parameter and turning those into proper ``protoc`` Helm charts that can be pushed to Tiller. ''' @classmethod def from_chart_doc(cls, chart): ''' Returns a ChartBuilder defined by an Armada Chart doc. :param chart: Armada Chart doc for which to build the Helm chart. ''' name = chart['metadata']['name'] chart_data = chart[const.KEYWORD_DATA] source_dir = chart_data.get('source_dir') source_directory = os.path.join(*source_dir) dependencies = chart_data.get('dependencies') # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(chart['schema']) if schema_info.version < 2: fix_tpl_name = False else: fix_tpl_name = True if dependencies is not None: dependency_builders = [] for chart_dep in dependencies: builder = ChartBuilder.from_chart_doc(chart_dep) dependency_builders.append(builder) return cls( name, source_directory, dependency_builders, fix_tpl_name=fix_tpl_name) return cls.from_source( name, source_directory, fix_tpl_name=fix_tpl_name) @classmethod def from_source(cls, name, source_directory, fix_tpl_name=False): ''' Returns a ChartBuilder, which gets it dependencies from within the Helm chart itself. :param name: A name to use for the chart. :param source_directory: The source directory of the Helm chart. ''' dependency_builders = [] charts_dir = os.path.join(source_directory, 'charts') if os.path.isdir(charts_dir): for f in os.scandir(charts_dir): if not f.is_dir(): # TODO: Support ".tgz" dependency charts. # Ignore regular files. continue # Ignore directories that start with "." or "_". if re.match(r'^[._]', f.name): continue builder = ChartBuilder.from_source( f.name, f.path, fix_tpl_name=fix_tpl_name) dependency_builders.append(builder) return cls(name, source_directory, dependency_builders, fix_tpl_name=fix_tpl_name) def __init__( self, name, source_directory, dependency_builders, fix_tpl_name=False): ''' :param name: A name to use for the chart. :param source_directory: The source directory of the Helm chart. :param dependency_builders: ChartBuilders to use to build the Helm chart's dependency charts. ''' self.name = name self.source_directory = source_directory self.dependency_builders = dependency_builders self.fix_tpl_name = fix_tpl_name # cache for generated protoc chart object self._helm_chart = None # load ignored files from .helmignore if present self.ignored_files = self.get_ignored_files() def get_ignored_files(self): '''Load files to ignore from .helmignore if present.''' try: ignored_files = [] if os.path.exists(os.path.join(self.source_directory, '.helmignore')): with open(os.path.join(self.source_directory, '.helmignore')) as f: ignored_files = f.readlines() return [filename.strip() for filename in ignored_files] except Exception: raise chartbuilder_exceptions.IgnoredFilesLoadException() def ignore_file(self, filename): '''Returns whether a given ``filename`` should be ignored. :param filename: Filename to compare against list of ignored files. :returns: True if file matches an ignored file wildcard or exact name, False otherwise. ''' for ignored_file in self.ignored_files: if (ignored_file.startswith('*') and filename.endswith(ignored_file.strip('*'))): return True elif ignored_file == filename: return True return False def get_metadata(self): '''Extract metadata from Chart.yaml to construct an instance of :class:`hapi.chart.metadata_pb2.Metadata`. ''' try: with open(os.path.join(self.source_directory, 'Chart.yaml')) as f: chart_yaml = yaml.safe_load(f.read().encode('utf-8')) except Exception: raise chartbuilder_exceptions.MetadataLoadException() # Construct Metadata object. return Metadata( description=chart_yaml.get('description'), name=chart_yaml.get('name'), version=chart_yaml.get('version')) def get_files(self): ''' Return (non-template) files in this chart. Non-template files include all files *except* Chart.yaml, values.yaml, values.toml, and any file nested under charts/ or templates/. The only exception to this rule is charts/.prov The class :class:`google.protobuf.any_pb2.Any` is wrapped around each file as that is what Helm uses. For more information, see: https://github.com/kubernetes/helm/blob/fa06dd176dbbc247b40950e38c09f978efecaecc/pkg/chartutil/load.go :returns: List of non-template files. :rtype: List[:class:`google.protobuf.any_pb2.Any`] ''' files_to_ignore = ['Chart.yaml', 'values.yaml', 'values.toml'] non_template_files = [] def _append_file_to_result(root, rel_folder_path, file): abspath = os.path.abspath(os.path.join(root, file)) relpath = os.path.join(rel_folder_path, file) encodings = ('utf-8', 'latin1') unicode_errors = [] for encoding in encodings: try: with open(abspath, 'r') as f: file_contents = f.read().encode(encoding) except OSError as e: LOG.debug( 'Failed to open and read file %s in the helm ' 'chart directory.', abspath) raise chartbuilder_exceptions.FilesLoadException( file=abspath, details=e) except UnicodeError as e: LOG.debug( 'Attempting to read %s using encoding %s.', abspath, encoding) msg = "(encoding=%s) %s" % (encoding, str(e)) unicode_errors.append(msg) else: break if len(unicode_errors) == 2: LOG.debug( 'Failed to read file %s in the helm chart directory.' ' Ensure that it is encoded using utf-8.', abspath) raise chartbuilder_exceptions.FilesLoadException( file=abspath, clazz=unicode_errors[0].__class__.__name__, details='\n'.join(e for e in unicode_errors)) non_template_files.append( Any(type_url=relpath, value=file_contents)) for root, dirs, files in os.walk(self.source_directory): relfolder = os.path.split(root)[-1] rel_folder_path = os.path.relpath(root, self.source_directory) if not any(root.startswith(os.path.join(self.source_directory, x)) for x in ['templates', 'charts']): for file in files: if (file not in files_to_ignore and file not in non_template_files): _append_file_to_result(root, rel_folder_path, file) elif relfolder == 'charts' and '.prov' in files: _append_file_to_result(root, rel_folder_path, '.prov') return non_template_files def get_values(self): '''Return the chart's (default) values.''' # create config object representing unmarshaled values.yaml if os.path.exists(os.path.join(self.source_directory, 'values.yaml')): with open(os.path.join(self.source_directory, 'values.yaml')) as f: raw_values = f.read() else: LOG.warn( "No values.yaml in %s, using empty values", self.source_directory) raw_values = '' return Config(raw=raw_values) def get_templates(self): '''Return all the chart templates. Process all files in templates/ as a template to attach to the chart, building a :class:`hapi.chart.template_pb2.Template` object. ''' chart_name = self.name templates = [] tpl_dir = os.path.join(self.source_directory, 'templates') if not os.path.exists(tpl_dir): LOG.warn( "Chart %s has no templates directory. " "No templates will be deployed", chart_name) for root, _, files in os.walk(tpl_dir, topdown=True): for tpl_file in files: tname = os.path.relpath( os.path.join(root, tpl_file), # For v1 compatibility, name template relative to template # dir, for v2 fix the name to be relative to the chart root # to match Helm CLI behavior. self.source_directory if self.fix_tpl_name else tpl_dir) # NOTE: If the template name is fixed (see above), then this # also fixes the path passed here, which could theoretically # affect which files get ignored, though unlikely. if self.ignore_file(tname): LOG.debug('Ignoring file %s', tname) continue with open(os.path.join(root, tpl_file)) as f: templates.append( Template(name=tname, data=f.read().encode())) return templates def get_helm_chart(self): '''Return a Helm chart object. Constructs a :class:`hapi.chart.chart_pb2.Chart` object from the ``chart`` intentions, including all dependencies. ''' if not self._helm_chart: self._helm_chart = self._get_helm_chart() return self._helm_chart def _get_helm_chart(self): LOG.info( "Building chart %s from path %s", self.name, self.source_directory) dependencies = [] for dep_builder in self.dependency_builders: LOG.info( "Building dependency chart %s for chart %s.", dep_builder.name, self.name) try: dependencies.append(dep_builder.get_helm_chart()) except Exception: raise chartbuilder_exceptions.DependencyException(self.name) try: helm_chart = Chart( metadata=self.get_metadata(), templates=self.get_templates(), dependencies=dependencies, values=self.get_values(), files=self.get_files()) except Exception as e: raise chartbuilder_exceptions.HelmChartBuildException( self.name, details=e) return helm_chart def dump(self): '''Dumps a chart object as a serialized string so that we can perform a diff. It recurses into dependencies. ''' return self.get_helm_chart().SerializeToString()