app-gen-tool/app-gen.py
Mingyuan Qi ad4c2b5d13 Add user application generation tool
This commit adds the user app gen tool for customer application
development. This tool completely decouples app development
from stx build, which means the app developers no longer need
to fetch stx code/build tool nor to build app by stx build system.

Features:

1. One command to package chart, generate manifest, checksum and
   package app.
2. Supports local dir, git repo and tarball as chart source.
3. The app manifest abstracts a few important fields from armada
   schema for user to lower the learning curve of armada.
4. Static value overrides allowed in app manifest

Story: 2006974
Task: 37704

Change-Id: I25a85e9fd7bdd0130041499eb9b1fc1350a63756
Signed-off-by: Mingyuan Qi <mingyuan.qi@intel.com>
2019-12-09 02:53:49 +00:00

613 lines
23 KiB
Python

import yaml
import os
import sys, getopt, getpass
import subprocess
import hashlib
import tarfile
import re
import shutil
from urllib import request
SCHEMA_CHART_TEMPLATE = 'template/armada-chart.template'
SCHEMA_CHARTGROUP_TEMPLATE = 'template/armada-chartgroup.template'
SCHEMA_MANIFEST_TEMPLATE = 'template/armada-manifest.template'
BIN_FETCH_CHART_INFO = 'bin/fetch_chart_info.sh'
TEMP_USER_DIR = '/tmp/' + getpass.getuser() + '/'
# Temp app work dir to hold git repo and upstream tarball
# TEMP_APP_DIR = TEMP_USER_DIR/appName
TEMP_APP_DIR = ''
APP_GEN_PY_PATH = os.path.split(os.path.realpath(__file__))[0]
def to_camel_case(s):
return s[0].lower() + s.title().replace('_','')[1:] if s else s
class ArmadaApplication:
def __init__(self, app_data):
# Initialize application config
self._armada_app = {}
# 'appName', 'namespace', 'version' are checked in check_manifest()
self._armada_app['appName'] = app_data['appName']
self._armada_app['namespace'] = app_data['namespace']
self._armada_app['version'] = app_data['version']
# Initialize manifest
self._armada_manifest = app_data['manifest']
# Initialize chartgroup
self._armada_chartgroup = app_data['chartGroup']
# Initialize chart
self._armada_chart = app_data['chart']
# add namespace and prefix to each chart
# 'namespace', 'releasePrefix' are checked in check_manifest()
for i in range(len(self._armada_chart)):
self._armada_chart[i]['namespace'] = self._armada_app['namespace']
self._armada_chart[i]['releasePrefix'] = self._armada_manifest['releasePrefix']
# TODO: Validate values
def _validate_app_values(self, app_data):
return True
# TODO: Validate values
def _validate_manifest_values(self, manifest_data):
return True
# TODO: Validate values
def _validate_chartgroup_values(self, chartgroup_data):
return True
# TODO: Validate values
def _validate_chart_values(self, chart_data):
return True
def _validate_app_attributes(self):
if not self._validate_app_values(self._armada_app):
return False
if not self._validate_manifest_values(self._armada_manifest):
return False
if not self._validate_chartgroup_values(self._armada_chartgroup):
return False
if not self._validate_chart_values(self._armada_chart):
return False
return True
def get_app_name(self):
return self._armada_app['appName']
def _package_helm_chart(self, chart):
path = chart['path']
# lint helm chart
cmd_lint = ['helm', 'lint', path]
subproc = subprocess.run(cmd_lint, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
print(str(subproc.stdout, encoding = 'utf-8'))
else:
print(str(subproc.stderr, encoding = 'utf-8'))
return False
# package helm chart
cmd_package = ['helm', 'package', path, '--save=false', \
'--destination=' + self._armada_app['outputChartDir']]
subproc = subprocess.run(cmd_package, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
output = str(subproc.stdout, encoding = 'utf-8')
print(output)
# capture tarball name
for words in output.split('/'):
if 'tgz' in words:
chart['tarballName'] = words.rstrip('\n')
else:
print(subproc.stderr)
return False
return True
# pyyaml does not support writing yaml block with initial indent
# add initial indent for yaml block substitution
def _write_yaml_to_manifest(self, key, src, init_indent):
target = {}
# add heading key
target[key] = src
lines = yaml.safe_dump(target).split('\n')
# remove ending space
lines.pop()
indents = ' ' * init_indent
for i in range(len(lines)):
lines[i] = indents + lines[i]
# restore ending '\n'
return '\n'.join(lines) + '\n'
def _substitute_values(self, in_line, dicts):
out_line = in_line
pattern = re.compile('\$.+?\$')
results = pattern.findall(out_line)
if results:
for result in results:
result_word = result.strip('$').split('%')
value_key = result_word[0]
value_default = ''
if len(result_word) > 1:
value_default = result_word[1]
# underscore case to camel case
value = to_camel_case(value_key)
if value in dicts:
out_line = out_line.replace(result, str(dicts[value]))
elif value_default:
out_line = out_line.replace(result, value_default)
if out_line == in_line:
return out_line, False
else:
return out_line, True
def _substitute_blocks(self, in_line, dicts):
out_line = in_line
result = re.search('@\S+\|\d+@',out_line)
if result:
block_key = result.group().strip('@').split('|')
key = block_key[0].lower()
indent = int(block_key[1])
if key in dicts:
out_line = self._write_yaml_to_manifest(key, dicts[key], indent)
else:
out_line = ''
return out_line
# Fetch info from helm chart to fill
# the values that needs to be substituted
# Below info are needed:
# - waitLabelKey
# - chartArcname
#
def _fetch_info_from_chart(self, chart_idx):
a_chart = self._armada_chart[chart_idx]
bin_fetch_script = APP_GEN_PY_PATH + '/' + BIN_FETCH_CHART_INFO
# waitLabelKey
# search for the key of label which indicates '.Release.Name'
# within deployment, statefulset, daemonset yaml file
cmd = [bin_fetch_script, 'waitlabel', a_chart['path']]
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
output = str(subproc.stdout, encoding = 'utf-8')
if output.strip():
a_chart['waitLabelKey'] = output.strip()
if 'waitLabelKey' not in a_chart:
print("The label which indicates .Release.Name is not found in %s" % a_chart['name'])
return False
# chartArcname is the helm chart name in Chart.yaml
# it is used as tarball arcname during helm package
cmd = [bin_fetch_script, 'chartname', a_chart['path']]
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
output = str(subproc.stdout, encoding = 'utf-8')
if output.strip():
a_chart['chartArcname'] = output.strip()
if 'chartArcname' not in a_chart:
print("The name within Chart.yaml of chart %s folder is not found" % a_chart['name'])
return False
return True
# Sub-process of app generation
# lint and package helm chart
# TODO: sub-chart dependency check
#
def _gen_helm_chart_tarball(self, chart):
ret = False
path = ''
print('Processing chart %s...' % chart['name'])
# check pathtype of the chart
if chart['_pathType'] is 'git':
gitname = ''
# download git
if not os.path.exists(TEMP_APP_DIR):
os.makedirs(TEMP_APP_DIR)
# if the git folder exists, check git name and use that folder
# otherwise git clone from upstream
if not os.path.exists(TEMP_APP_DIR + chart['_gitname']):
saved_pwd = os.getcwd()
os.chdir(TEMP_APP_DIR)
cmd = ['git', 'clone', chart['path']]
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding = 'utf-8')
print(output)
print('Error: git clone %s failed' % chart['_gitname'])
os.chdir(saved_pwd)
return False
os.chdir(saved_pwd)
else:
# git pull to ensure folder up-to-date
saved_pwd = os.getcwd()
os.chdir(TEMP_APP_DIR + chart['_gitname'])
cmd = ['git', 'pull']
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding = 'utf-8')
print(output)
print('Error: git pull for %s failed' % chart['_gitname'])
os.chdir(saved_pwd)
return False
os.chdir(saved_pwd)
path = TEMP_APP_DIR + chart['_gitname'] + '/' + chart['subpath']
elif chart['_pathType'] is 'tarball':
if not os.path.exists(TEMP_APP_DIR):
os.makedirs(TEMP_APP_DIR)
try:
# check whether it's a url or local tarball
if not os.path.exists(chart['path']):
# download tarball
tarpath = TEMP_APP_DIR + chart['_tarname'] + '.tgz'
if not os.path.exists(tarpath):
res = request.urlopen(chart['path'])
with open(tarpath, 'wb') as f:
f.write(res.read())
else:
tarpath = chart['path']
# extract tarball
chart_tar = tarfile.open(tarpath, 'r:gz')
chart_files = chart_tar.getnames()
# get tar arcname for packaging helm chart process
# TODO: compatible with the case that there is no arcname
chart['_tarArcname'] = chart_files[0].split('/')[0]
if not os.path.exists(chart['_tarArcname']):
for chart_file in chart_files:
chart_tar.extract(chart_file, TEMP_APP_DIR)
chart_tar.close()
except Exception as e:
print('Error: %s' % e)
return False
path = TEMP_APP_DIR + chart['_tarArcname'] + '/' + chart['subpath']
elif chart['_pathType'] is 'dir':
path = chart['path']
# update chart path
# remove ending '/'
chart['path'] = path.rstrip('/')
# lint and package
ret = self._package_helm_chart(chart)
return ret
# Sub-process of app generation
# generate application manifest file
#
def _gen_armada_manifest(self):
# check manifest file existance
manifest_file = self._armada_app['outputDir'] + '/' + self._armada_app['appName'] + '.yaml'
if os.path.exists(manifest_file):
os.remove(manifest_file)
# update schema path to abspath
chart_template = APP_GEN_PY_PATH + '/' + SCHEMA_CHART_TEMPLATE
chartgroup_template = APP_GEN_PY_PATH + '/' + SCHEMA_CHARTGROUP_TEMPLATE
manifest_template = APP_GEN_PY_PATH + '/' + SCHEMA_MANIFEST_TEMPLATE
# generate chart schema
try:
with open(chart_template, 'r') as f:
chart_schema = f.readlines()
except IOError:
print('File %s not found' % chart_template)
return False
with open(manifest_file, 'a') as f:
# iterate each armada_chart
for idx in range(len(self._armada_chart)):
a_chart = self._armada_chart[idx]
# fetch chart specific info
if not self._fetch_info_from_chart(idx):
return False
for line in chart_schema:
# substitute template values to chart values
out_line, substituted = self._substitute_values(line, a_chart)
if not substituted:
# substitute template blocks to chart blocks
out_line = self._substitute_blocks(line, a_chart)
f.write(out_line)
# generate chartgroup schema
try:
with open(chartgroup_template, 'r') as f:
chartgroup_schema = f.readlines()
except IOError:
print('File %s not found' % chartgroup_template)
return False
with open(manifest_file, 'a') as f:
# iterate each chartgroup
for chartgroup in self._armada_chartgroup:
for line in chartgroup_schema:
# substitute template values to chartgroup values
out_line, substituted = self._substitute_values(line, chartgroup)
if not substituted:
# substitute template blocks to chartgroup blocks
out_line = self._substitute_blocks(line, chartgroup)
f.write(out_line)
# generate manifest schema
try:
with open(manifest_template, 'r') as f:
manifest_schema = f.readlines()
except IOError:
print('File %s not found' % manifest_template)
return False
with open(manifest_file, 'a') as f:
# only one manifest in an application
manifest = self._armada_manifest
# substitute values
for line in manifest_schema:
# substitute template values to manifest values
out_line, substituted = self._substitute_values(line, manifest)
if not substituted:
# substitute template blocks to manifest blocks
out_line = self._substitute_blocks(line, manifest)
f.write(out_line)
return True
# Sub-process of app generation
# generate application metadata
#
def _gen_metadata(self):
# check metadata file existance
metadata_file = self._armada_app['outputDir'] + '/metadata.yaml'
if os.path.exists(metadata_file):
os.remove(metadata_file)
with open(metadata_file, 'a') as f:
f.write('app_name: ' + self._armada_app['appName'] + '\n')
f.write('app_version: ' + self._armada_app['version'] + '\n')
return True
def _gen_md5(self, in_file):
with open(in_file, 'rb') as f:
out_md5 = hashlib.md5(f.read()).hexdigest()
return out_md5
# Sub-process of app generation
# generate application checksum file and tarball
#
def _gen_checksum_and_app_tarball(self):
store_cwd = os.getcwd()
os.chdir(self._armada_app['outputDir'])
# gen checksum
# check checksum file existance
checksum_file = 'checksum.md5'
if os.path.exists(checksum_file):
os.remove(checksum_file)
app_files = []
for parent, dirnames, filenames in os.walk('./'):
for filename in filenames:
app_files.append(os.path.join(parent, filename))
with open(checksum_file, 'a') as f:
for target_file in app_files:
f.write(self._gen_md5(target_file) + ' ' + target_file + '\n')
app_files.append('./' + checksum_file)
# gen application tarball
tarname = self._armada_app['appName'] + '-' + self._armada_app['version'] + '.tgz'
t = tarfile.open(tarname, 'w:gz')
for target_file in app_files:
t.add(target_file)
t.close()
os.chdir(store_cwd)
return tarname
# Generate armada application, including:
# 1. helm chart tarballs
# 2. armada manifest
# 3. metadata file
# 4. checksum file
# 5. application tarball
#
def gen_app(self, output_dir, overwrite):
ret = False
if not self._validate_app_attributes():
print('Error: Some of the app attributes are not valid!')
return ret
self._armada_app['outputDir'] = output_dir
self._armada_app['outputChartDir'] = output_dir + '/charts'
if not os.path.exists(self._armada_app['outputDir']):
os.makedirs(self._armada_app['outputDir'])
elif overwrite:
shutil.rmtree(self._armada_app['outputDir'])
else:
print('Output folder %s exists, please remove it or use --overwrite.' % self._armada_app['outputDir'])
return ret
if not os.path.exists(self._armada_app['outputChartDir']):
os.makedirs(self._armada_app['outputChartDir'])
# 1. Generating helm chart tarball
for chart in self._armada_chart:
ret = self._gen_helm_chart_tarball(chart)
if ret:
print('Helm chart %s tarball generated!' % chart['name'])
print('')
else:
print('Generating tarball for helm chart: %s error!' % chart['name'])
return ret
# 2. Generating armada manifest
ret = self._gen_armada_manifest()
if ret:
print('Armada manifest generated!')
else:
print('Armada manifest generation failed!')
return ret
# 3. Generating metadata file
ret = self._gen_metadata()
if ret:
print('Metadata generated!')
else:
print('Metadata generation failed!')
return ret
# 4&5. Generating checksum file and tarball
ret = self._gen_checksum_and_app_tarball()
if ret:
print('Checksum generated!')
print('App tarball generated at %s/%s' % (self._armada_app['outputDir'], ret))
print('')
else:
print('Checksum and App tarball generation failed!')
return ret
return ret
# For debug
def print_app_data(self):
print(self._armada_app)
print(self._armada_manifest)
print(self._armada_chartgroup)
print(self._armada_chart)
def parse_yaml(yaml_in):
yaml_data=''
try:
with open(yaml_in) as f:
yaml_data = yaml.safe_load(f)
except IOError:
print('Error: %s no found' % yaml_in )
except Exception as e:
print('Error: Invalid yaml file')
return yaml_data
def check_manifest(manifest_data):
# TODO: check more mandatory key/values in manifest yaml
# check app values
if 'appName' not in manifest_data:
print('Error: \'appName\' is missing.')
return False
if 'namespace' not in manifest_data:
print('Error: \'namespace\' is missing.')
return False
if 'version' not in manifest_data:
print('Error: \'version\' is missing.')
return False
# check manifest values
if 'manifest' not in manifest_data:
print('Error: \'manifest\'is missing.')
return False
if 'releasePrefix' not in manifest_data['manifest']:
print('Error: Manifest attribute \'releasePrefix\' is missing.')
return False
# check chartGroup values
if 'chartGroup' not in manifest_data:
print('Error: \'chartGroup\' is missing.')
return False
# check chart values
if 'chart' not in manifest_data:
print('Error: \'chart\' is missing.')
return False
for chart in manifest_data['chart']:
# check chart name
if 'name' not in chart:
print('Error: Chart attribute \'name\' is missing.')
return False
# check chart path, supporting: dir, git, tarball
if 'path' not in chart:
print('Error: Chart attribute \'path\' is missing in chart %s.' % chart['name'])
return False
else:
# TODO: To support branches/tags in git repo
if chart['path'].endswith('.git'):
if 'subpath' not in chart:
print('Error: Chart attribute \'subpath\' is missing in chart %s.' % chart['name'])
return False
chart['_pathType'] = 'git'
gitname = re.search('[^/]+(?=\.git$)',chart['path']).group()
if gitname:
chart['_gitname'] = gitname
else:
print('Error: Invalid \'path\' in chart %s.' % chart['name'])
print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported')
return False
elif chart['path'].endswith('.tar.gz') or chart['path'].endswith('.tgz'):
if 'subpath' not in chart:
print('Error: Chart attribute \'subpath\' is missing in chart %s.' % chart['name'])
return False
chart['_pathType'] = 'tarball'
tarname = re.search('[^/]+(?=\.tgz)|[^/]+(?=\.tar\.gz)',chart['path']).group()
if tarname:
chart['_tarname'] = tarname
else:
print('Error: Invalid \'path\' in chart %s.' % chart['name'])
print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported')
return False
else:
if not os.path.isdir(chart['path']):
print('Error: Invalid \'path\' in chart %s.' % chart['name'])
print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported')
return False
chart['_pathType'] = 'dir'
return True
def generate_app(file_in, out_folder, overwrite):
global TEMP_APP_DIR
app_data = parse_yaml(file_in)
if not app_data:
print('Parse yaml error')
return
if not check_manifest(app_data):
print('Application manifest is not valid')
return
armada_app = ArmadaApplication(app_data)
TEMP_APP_DIR = TEMP_USER_DIR + armada_app.get_app_name() + '/'
app_out = out_folder + '/' + armada_app.get_app_name()
armada_app.gen_app(app_out, overwrite)
def main(argv):
input_file = ''
output_folder = '.'
overwrite = False
try:
options, args = getopt.getopt(argv, 'hi:o:', \
['help', 'input==', 'output==', 'overwrite'])
except getopt.GetoptError:
sys.exit()
for option, value in options:
if option in ('-h', '--help'):
print('StarlingX User Application Generator')
print('')
print('Usage:')
print(' python app-gen.py [Option]')
print('')
print('Options:')
print(' -i, --input yaml_file generate app from yaml_file')
print(' -o, --output folder generate app to output folder')
print(' --overwrite overwrite the output dir')
print(' -h, --help this help')
if option in ('--overwrite'):
overwrite = True
if option in ('-i', '--input'):
input_file = value
if option in ('-o', '--output'):
output_folder = value
if not os.path.isfile(os.path.abspath(input_file)):
print('Error: input file not found')
sys.exit()
if input_file:
generate_app(os.path.abspath(input_file), os.path.abspath(output_folder), overwrite)
if __name__ == '__main__':
main(sys.argv[1:])