starlingx/app-gen-tool
Code Issues Proposed changes

Refactor support for tar and git helm-charts

Browse Source 
This commit aims to support helm-charts passed
as dir, git url or as a .tar package and unit
tests for these functionalities.
It also removes some of the remanescent Armada
sections from the app_manifest.yaml that i forgot
to remove in review 902843 and correct some
little flake8 warnings.

Test Plan:
PASS - Helm charts passed as dir in the
app_manifest path are working as expected
PASS - Helm charts passed as git url in the
app_manifest path are working as expected
PASS - Helm charts passed as tar packages in the
app_manifest path are working as expected

Story: 2010937
Task: 49130
Task: 49131

Change-Id: I1fc0e98f731c9a43f742b94d2044c57291876fc0
Signed-off-by: Tomás Barros <tomas.barros@encora.com>
This commit is contained in:
Tomás Barros 2023-12-14 19:28:56 -03:00
parent 9cd8915edc
commit cac2937bef
22 changed files with 457 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/launch.json vendored
View File

@ -13,7 +13,7 @@
"console": "integratedTerminal",
"justMyCode": true,
"args": [
"--input=${workspaceFolder}/example/app-test-adminer-1.yaml",
"--input=${workspaceFolder}/example/app-test-example.yaml",
"--output=${workspaceFolder}/example/output",
"--overwrite"
]
@ -29,7 +29,7 @@
"console": "integratedTerminal",
"justMyCode": true,
"args": [
"--input=${workspaceFolder}/example/app-test-adminer-1.yaml",
"--input=${workspaceFolder}/example/app-test-example.yaml",
"--output=${workspaceFolder}/example/output",
"--overwrite"
]

17
app_manifest.yaml
View File

@ -8,21 +8,8 @@ appManifestFile-config:
- name: # required
version: # required
path: # required
chartGroup: # required
chartGroup: # required name
# add more if you have more than one chart
chartGroup:
- name: # required
description: # required for Armada
sequenced: # required for Armada <true/false>
chart_names:
- # required
- # optional
# add more if you have more than one chartgroup for your Armada app
manifest:
name: # required for Armada
releasePrefix: # required for Armada
## For Armada packaging the sections bellow are not necessary.
#################################################
## App Metadata Configuration
@ -79,7 +66,7 @@ metadataFile-config:
setupFile-config:
metadata:
author: # required
author-email: # required
author_email: # required
url: # required
classifier: # required
- # required

BIN
example/adminer-0.2.1.tgz
View File

Binary file not shown.

46
example/app-test-adminer-1.yaml
View File

@ -1,46 +0,0 @@
---
## App Manifest Configuration
appManifestFile-config:
appName: app-adminer
appVersion: 1.0-1
namespace: default
chart:
- name: adminer
version: 0.2.1
path: ./example/adminer
chartGroup:
- name: adminer
chart_names:
- adminer
#################################################
## App Metadata Configuration
# for further details about possible configurations on this file, please
# visit the link: https://wiki.openstack.org/wiki/StarlingX/Containers/StarlingXAppsInternals#metadata.yaml
metadataFile-config:
# the following configurations are optional
# uncomment and configure properly the ones you need for your application metadata
upgrades:
auto_update: true
supported_k8s_version:
minimum: 1.21.8
maximum: 1.26.1
k8s_upgrades:
auto_update: true
timing: pre
maintain_user_overrides: true
#################################################
## App Setup Configuration
# if you wish to see a setup.cfg example, please see the link
# https://opendev.org/starlingx/app-dell-storage/src/branch/master/python3-k8sapp-dell-storage/k8sapp_dell_storage/setup.cfg
setupFile-config:
metadata:
author: StarlingX
author_email: starlingx-discuss@lists.starlingx.io
url: https://www.starlingx.io/
classifier:
- "Environment :: OpenStack"

40
example/app-test-example.yaml Normal file
View File

@ -0,0 +1,40 @@
---
appManifestFile-config:
appName: app-test
appVersion: 1.0-1
namespace: default
chart:
- name: adminer
version: 0.2.1
path: https://github.com/cetic/helm-adminer.git # git example path
chartGroup: chartGroup-test
- name: tempo-vulture
version: 0.4.1
path: app-gen-tool/example/tempo-vulture/tempo-vulture-0.4.1.tgz # tar example path
chartGroup: chartGroup-test
- name: poc-starlingx
version: 1.5.2
path: app-gen-tool/example/poc-starlingx-messages # dir example path
chartGroup: chartGroup-test
metadataFile-config:
upgrades:
auto_update: true
supported_k8s_version:
minimum: 1.21.8
maximum: 1.26.1
k8s_upgrades:
auto_update: true
timing: pre
maintain_user_overrides: true
setupFile-config:
metadata:
author: StarlingX
author_email: starlingx-discuss@lists.starlingx.io
url: https://www.starlingx.io/
classifier:
- "Environment :: OpenStack"

23
example/poc-starlingx-messages/.helmignore Normal file
View File

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

23
example/poc-starlingx-messages/Chart.yaml Normal file
View File

@ -0,0 +1,23 @@
apiVersion: v2
name: poc-starlingx
description: A very very very basic messaging exchange app
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.5.2
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
appVersion: 1.5.2

23
example/poc-starlingx-messages/templates/deployment.yaml Normal file
View File

@ -0,0 +1,23 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.kube.name }}
labels:
app: {{ .Values.kube.name }}
spec:
replicas: {{ .Values.kube.replicas }}
selector:
matchLabels:
app: {{ .Values.kube.name }}
template:
metadata:
labels:
app: {{ .Values.kube.name }}
spec:
containers:
- name: {{ .Values.kube.name }}
image: docker.io/brunomuniz/poc-starlingx:{{ .Values.image.tag }}
env:
{{- toYaml .Values.env | nindent 10 }}
ports:
- containerPort: {{ .Values.image.containerPort }}

14
example/poc-starlingx-messages/templates/service.yaml Normal file
View File

@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.kube.name }}
labels:
app: {{ .Values.kube.name }}
spec:
type: NodePort
ports:
- port: 8100
targetPort: {{ .Values.image.containerPort }}
nodePort: {{ .Values.kube.port }}
selector:
app: {{ .Values.kube.name }}

22
example/poc-starlingx-messages/values.yaml Normal file
View File

@ -0,0 +1,22 @@
# Default values for helm-chart.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
env:
- name: MODE
value: node
- name: SERVER
value: 127.0.0.1:8000
- name: PORT
value: "8000"
image:
repository: docker.io/brunomuniz/poc-starlingx
tag: 1.5.2
containerPort: 8000
kube:
port: 31234
replicas: 1
secret: my-docker-reg-secret
name: poc-starlingx

BIN
example/tempo-vulture/tempo-vulture-0.4.1.tgz Normal file
View File

Binary file not shown.

270
stx-app-generator/stx-app-generator/app_gen_tool/application.py
View File

@ -9,9 +9,11 @@ import yaml
from abc import ABC
from abc import abstractmethod
from urllib import request
from app_gen_tool.common import extract
from app_gen_tool.common import get_chart_from_git
from app_gen_tool.common import to_camel_case
from app_gen_tool.common import transform_rel_path_into_abs_path
from app_gen_tool import constants
@ -28,7 +30,8 @@ class Application(ABC):
# Initialize application config
self._app: dict = {}
self._app = app_data["appManifestFile-config"]
self._temp_app_dir = os.path.join(constants.TEMP_USER_DIR, self.get_app_name())
self._temp_app_dir = os.path.join(constants.TEMP_USER_DIR,
self.get_app_name())
self.output_folder: str = output_folder
self.APP_NAME = self._app["appName"]
@ -42,15 +45,24 @@ class Application(ABC):
= app_data["appManifestFile-config"]["chart"][0]["chartGroup"]
# Create empty list that will contain all the charts
self._listcharts = {}
self._listcharts["chart_names"] = []
self._listcharts["namespace"] = self._app["namespace"]
self._listcharts = {
"chart_names": [],
"namespace": self._app["namespace"]
}
# Initialize chart
self._chart = app_data["appManifestFile-config"]["chart"]
for i in range(len(self._chart)):
self._chart[i]["namespace"] = self._app["namespace"]
self._listcharts["chart_names"].append(self._chart[i]["name"])
self._chart[i]['pathType'] = self._define_chart_pathtype(i)
if self._chart[i]['pathType'] == 'git':
self._process_git_helm_charts(i)
else:
self._chart[i]['path'] = transform_rel_path_into_abs_path(os.getcwd(),
self._chart[i]['path'])
if self._chart[i]['pathType'] == 'tarball':
self._extract_helm_tars_to_temp_dir(chart_index=i)
ret = self._get_values_file(self._chart[i]['path'])
self._chart[i]['values'] = ret
@ -77,41 +89,42 @@ class Application(ABC):
chart["version"],
)
if chart["_pathType"] == "dir":
try:
chart_metadata_f = open(chart_yaml_file, "r", encoding="utf-8")
except Exception as e:
print(f"ERROR: {e}")
sys.exit(1)
chart_file_lines = chart_metadata_f.readlines()
chart_file_lines = [
cline
for cline in chart_file_lines
if len(cline) > 0 and cline[0] != "#"
]
chart_metadata_f.close()
for line in chart_file_lines:
line = line.rstrip("\n")
line_data = line.split()
if not line_data:
continue
if "name:" in line_data[0]:
chart_file_data["name"] = line_data[-1]
elif "version:" in line_data[0]:
chart_file_data["version"] = line_data[-1]
try:
chart_metadata_f = open(chart_yaml_file, "r",
encoding="utf-8")
except Exception as e:
print(f"ERROR: {e}")
sys.exit(1)
chart_file_lines = chart_metadata_f.readlines()
chart_file_lines = [
cline
for cline in chart_file_lines
if len(cline) > 0 and cline[0] != "#"
]
chart_metadata_f.close()
for line in chart_file_lines:
line = line.rstrip("\n")
line_data = line.split()
if not line_data:
continue
if "name:" in line_data[0]:
chart_file_data["name"] = line_data[-1]
elif "version:" in line_data[0]:
chart_file_data["version"] = line_data[-1]
# To-do chart type different from dir
for key in manifest_data:
err_str = ""
if key not in chart_file_data:
err_str = \
f'"{key}" is present in app-manifest.yaml but not in {chart_yaml_file}'
raise KeyError(err_str)
if manifest_data[key] != chart_file_data[key]:
err_str = \
f'"{key}" has different values ' \
f'in app-manifest.yaml and {chart_yaml_file}'
raise ValueError(err_str)
# To-do chart type different from dir
for key in manifest_data:
err_str = ""
if key not in chart_file_data:
err_str = \
f'"{key}" is present in app-manifest.yaml but not in' \
f'{chart_yaml_file}'
raise KeyError(err_str)
if manifest_data[key] != chart_file_data[key]:
err_str = \
f'"{key}" has different values ' \
f'in app-manifest.yaml and {chart_yaml_file}'
raise ValueError(err_str)
def get_app_name(self):
"""Return name of app.
@ -121,7 +134,8 @@ class Application(ABC):
"""
return self._app["appName"]
def _package_helm_chart(self, chart, chart_dir) -> bool:
@staticmethod
def _package_helm_chart(chart, chart_dir) -> bool:
"""Sub-process of app generation. Generate application helm-charts tarball.
Args:
@ -151,7 +165,8 @@ class Application(ABC):
return False
# package helm chart
cmd_package = ["helm", "package", chart["path"], "--destination=" + chart_dir]
cmd_package = ["helm", "package", chart["path"],
"--destination=" + chart_dir]
subproc = subprocess.run(
cmd_package,
env=os.environ.copy(),
@ -172,7 +187,8 @@ class Application(ABC):
return True
def _get_values_file(self, path):
@staticmethod
def _get_values_file(path):
"""Retrieve the content of a values file and saves them in a dictionary.
Args:
@ -181,6 +197,7 @@ class Application(ABC):
Returns:
dict: Data from the values.yaml found in the Helm chart path.
"""
values_yaml_path = os.path.join(path, "values.yaml")
try:
with open(values_yaml_path, "r", encoding='utf-8') as f:
@ -196,7 +213,8 @@ class Application(ABC):
# pyyaml does not support writing yaml block with initial indent
# add initial indent for yaml block substitution
def _write_yaml_to_manifest(self, key, src, init_indent):
@staticmethod
def _write_yaml_to_manifest(key, src, init_indent):
"""Return the values of a given key in a yaml file.
Args:
@ -207,9 +225,8 @@ class Application(ABC):
Returns:
str: Values with indentation and break lines.
"""
target = {}
target = {key: src}
# add heading key
target[key] = src
lines = yaml.safe_dump(target, sort_keys=False).split("\n")
# remove ending space and first line
lines.pop()
@ -284,7 +301,8 @@ class Application(ABC):
key = block_key[0].lower()
indent = int(block_key[1])
if key in dicts:
out_line = self._write_yaml_to_manifest(key, dicts[key], indent)
out_line = self._write_yaml_to_manifest(key, dicts[key],
indent)
else:
out_line = ""
return out_line
@ -303,102 +321,7 @@ class Application(ABC):
ret = False
chart_path = ""
print(f'Processing chart {chart["name"]}...')
# check pathtype of the chart
if chart["_pathType"] == "git":
# download git
print("Processing chart _pathType git...")
if not os.path.exists(self._temp_app_dir):
os.makedirs(self._temp_app_dir)
# if the git folder exists, check git name and use that folder
# otherwise git clone from upstream
if not os.path.exists(self._temp_app_dir + chart["_gitname"]):
saved_pwd = os.getcwd()
os.chdir(self._temp_app_dir)
cmd = ["git", "clone", chart["path"]]
subproc = subprocess.run(
cmd,
env=os.environ.copy(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding="utf-8")
print(output)
print(f'Error: git clone {chart["_gitname"]} failed')
os.chdir(saved_pwd)
return False
os.chdir(saved_pwd)
else:
# git pull to ensure folder up-to-date
saved_pwd = os.getcwd()
os.chdir(self._temp_app_dir + chart["_gitname"])
cmd = ["git", "pull"]
subproc = subprocess.run(
cmd,
env=os.environ.copy(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding="utf-8")
print(output)
print(f'Error: git pull for {chart["_gitname"]} failed')
os.chdir(saved_pwd)
return False
os.chdir(saved_pwd)
chart_path = os.path.join(
self._temp_app_dir, chart["_gitname"], chart["subpath"]
)
elif chart["_pathType"] == "tarball":
print("Processing chart _pathType tarball...")
if not os.path.exists(self._temp_app_dir):
os.makedirs(self._temp_app_dir)
try:
# check whether it's a url or local tarball
if not os.path.exists(chart["path"]):
# download tarball
tarpath = self._temp_app_dir + chart["_tarname"] + ".tgz"
if not os.path.exists(tarpath):
res = request.urlopen(chart["path"]) # nosec
with open(tarpath, "wb") as f:
f.write(res.read())
else:
tarpath = chart["path"]
# extract tarball
chart_tar = tarfile.open(tarpath, "r:gz")
chart_files = chart_tar.getnames()
# get tar arcname for packaging helm chart process
# TODO: compatible with the case that there is no arcname
chart["_tarArcname"] = chart_files[0].split("/")[0]
if not os.path.exists(chart["_tarArcname"]):
for chart_file in chart_files:
chart_tar.extract(chart_file, self._temp_app_dir)
chart_tar.close()
except Exception as e:
print(f"Error: {e}")
return False
chart_path = os.path.join(
self._temp_app_dir, chart["_tarArcname"], chart["subpath"]
)
elif chart["_pathType"] == "dir":
print("Processing chart _pathType dir...")
chart_path = chart["path"]
chart_path = chart["path"]
# update chart path
# remove ending '/', if it exists...
chart["path"] = chart_path.rstrip("/")
@ -433,7 +356,8 @@ class Application(ABC):
# Sub-process of app generation
# generate application sha256 file
def _gen_sha256(self, in_file):
@staticmethod
def _gen_sha256(in_file):
"""Sub-process of app generation. Generate hash function of a file.
Args:
@ -478,7 +402,8 @@ class Application(ABC):
try:
with open(checksum_file, "a", encoding="utf-8") as f:
for target_file in sorted(app_files):
f.write(self._gen_sha256(target_file) + " *" + target_file + "\n")
f.write(self._gen_sha256(
target_file) + " *" + target_file + "\n")
except Exception as e:
print(f"Error: {e}")
@ -496,7 +421,62 @@ class Application(ABC):
os.chdir(store_cwd)
return tarname
# For debug
def _extract_helm_tars_to_temp_dir(self, chart_index):
"""Auxiliary function to extract helm charts passed as tar packages"""
try:
chart = self._chart[chart_index]
tarball_dir = chart['path']
tmp_rel_path = self._temp_app_dir
extract_path = os.path.join(tmp_rel_path, self.APP_NAME)
if not os.path.exists(extract_path):
os.makedirs(extract_path)
extract(tarball_dir, extract_path)
self._chart[chart_index]['tarball_path'] = \
chart['path']
self._chart[chart_index]['path'] = os.path.join(extract_path,
chart['name'])
except Exception as e:
print(f"ERROR: {e}")
sys.exit(1)
def _define_chart_pathtype(self, i: int):
"""Gets pathtype for each chart"""
chart = self._chart[i]
path = chart['path']
path_type = ''
if path.endswith('.tar') or path.endswith('.tar.gz') \
or path.endswith('.tgz'):
path_type = 'tarball'
elif path.endswith('.git'):
path_type = 'git'
else:
path_type = 'dir'
return path_type
def _process_git_helm_charts(self, chart_index: int):
self._chart[chart_index]['_gitname'] = re.search(r'[^/]+(?=\.git$)',
self._chart[
chart_index][
'path']).group()
chart = self._chart[chart_index]
if not os.path.exists(self._temp_app_dir):
os.makedirs(self._temp_app_dir)
# if the git folder exists, check git name and use that folder
# otherwise git clone from upstream
new_path = os.path.join(self._temp_app_dir,
chart['_gitname'])
_, saved_pwd = get_chart_from_git(
new_path, self._temp_app_dir,
chart['path'],
chart['_gitname']
)
self._chart[chart_index]['_gitUrl'] = chart['path']
self._chart[chart_index]['path'] = os.path.join(saved_pwd, new_path)
os.chdir(saved_pwd)
return True
def print_app_data(self):
"""Debug Print App Data."""
print(self._app)

82
stx-app-generator/stx-app-generator/app_gen_tool/common.py
View File

@ -1,4 +1,8 @@
"""Module for common methods accross entire codebase."""
import os
import subprocess
import sys
import tarfile
def to_camel_case(s: str) -> str:
@ -23,3 +27,81 @@ def uppercase_name(name: str) -> str:
str: String converted to uppercase.
"""
return name.upper().replace(" ", "_").replace("-", "_")
def extract(tar_path='.', extract_path='.'):
"""Extracts content from a .tar package stored locally or on the web"""
try:
if not os.path.exists(tar_path):
new_path = os.getcwd()
tar_path = transform_rel_path_into_abs_path(new_path, tar_path)
tar = tarfile.open(tar_path, 'r')
for item in tar:
tar.extract(item, extract_path)
if item.name.find(".tgz") != -1 or item.name.find(".tar") != -1:
extract(item.name, "./" + item.name[:item.name.rfind('/')])
except Exception as e:
print(f"Exception error {e} occurred during tar extraction!")
sys.exit(1)
def get_chart_from_git(new_path: str, temp_dir: str, git_url: str, git_name: str):
"""Auxiliary function to clone helm-charts from a git url
Returns a pair of type boolean, string.
The boolean indicates whether the git clone was successful or not.
The string indicates the saved pwd
"""
if not os.path.exists(new_path):
saved_pwd = os.getcwd()
os.chdir(temp_dir)
cmd = ["git", "clone", git_url]
subproc = subprocess.run(
cmd,
env=os.environ.copy(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding="utf-8")
print(output)
print(f'Error: git clone {git_name} failed')
os.chdir(saved_pwd)
return False, saved_pwd
os.chdir(saved_pwd)
else:
# git pull to ensure folder up-to-date
saved_pwd = os.getcwd()
os.chdir(new_path)
cmd = ["git", "pull"]
subproc = subprocess.run(
cmd,
env=os.environ.copy(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding="utf-8")
print(output)
print(f'Error: git pull for {git_name} failed')
os.chdir(saved_pwd)
return False, saved_pwd
return True, saved_pwd
def transform_rel_path_into_abs_path(abs_path: str, rel_path: str) -> str:
"""It transforms the rel_path into an absolute path"""
if rel_path[:2] == './':
rel_path = rel_path.replace('./', 'app-gen-tool/')
longest_common_path = ""
for i in range(len(rel_path)):
common_path = rel_path[:i + 1]
if common_path in abs_path:
longest_common_path = common_path
replace_index = abs_path.find(longest_common_path)
return abs_path[:replace_index] + rel_path

2
stx-app-generator/stx-app-generator/app_gen_tool/constants.py
View File

@ -13,5 +13,5 @@ FLUXCD_LIFECYCLE_TEMPLATE = os.path.join('templates_plugins', 'lifecycle.templat
# Other variables
BIN_FETCH_CHART_INFO = os.path.join('scripts', 'fetch_chart_info.sh')
TEMP_USER_DIR = os.path.join('tmp', getpass.getuser())
TEMP_USER_DIR = os.path.join('/tmp', getpass.getuser())
APP_GEN_PY_PATH = os.path.split(os.path.realpath(__file__))[0]

5
stx-app-generator/stx-app-generator/app_gen_tool/fluxcd.py
View File

@ -618,8 +618,9 @@ class FluxCD(Application):
for i, plug in enumerate(plugins_names):
out += (
f'\t{i+1:03d}_{plug} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.'
f'helm.{plug.replace("-","_")}'
f'\t{i + 1: 03d}_{plug} = k8sapp'
f'_{self.APP_NAME_WITH_UNDERSCORE}.'
f'helm.{plug.replace("-", "_")}'
)
out += f':{plug.replace("-", " ").title().replace(" ", "")}Helm\n'

50
stx-app-generator/stx-app-generator/app_gen_tool/generator.py
View File

@ -1,7 +1,6 @@
"""Generator Main Module."""
import argparse
import os
import re
import shutil
import sys
import yaml
@ -86,55 +85,6 @@ def check_manifest(manifest_data): # pylint: disable=too-many-return-statements
f'Error: Chart attribute \'path\' is missing in chart {chart["name"]}.'
)
return False
else:
# TODO: To support branches/tags in git repo
if chart['path'].endswith('.git'):
if 'subpath' not in chart:
print(
'Error: Chart attribute "subpath" is missing in '
f'chart {chart["name"]}.'
)
return False
chart['_pathType'] = 'git'
gitname = re.search(r'[^/]+(?=\.git$)', chart['path']).group()
if gitname:
chart['_gitname'] = gitname
else:
print(f'Error: Invalid \'path\' in chart {chart["name"]}.')
print(
' only "local dir", ".git", ".tar.gz", ".tgz" are supported'
)
return False
elif chart['path'].endswith('.tar.gz') or chart['path'].endswith('.tgz'):
if 'subpath' not in chart:
print(
'Error: Chart attribute "subpath" is missing in '
f'chart {chart["name"]}.'
)
return False
chart['_pathType'] = 'tarball'
tarname = \
re.search(
r'[^/]+(?=\.tgz)|[^/]+(?=\.tar\.gz)',
chart['path']
).group()
if tarname:
chart['_tarname'] = tarname
else:
print(f'Error: Invalid \'path\' in chart {chart["name"]}.')
print(
" only 'local dir', '.git', '.tar.gz', '.tgz' are supported"
)
return False
else:
if not os.path.isdir(chart['path']):
print(f'Error: Invalid \'path\' in chart {chart["name"]}.')
print(
" only 'local dir', '.git', '.tar.gz', '.tgz' are supported"
)
return False
chart['_pathType'] = 'dir'
return True

2
stx-app-generator/stx-app-generator/tests/unit/resources/app-test.yaml
View File

@ -7,7 +7,7 @@ appManifestFile-config:
chart:
- name: adminer
version: 0.2.1
path: ./stx-app-generator/stx-app-generator/tests/unit/resources/adminer/adminer
path: app-gen-tool/stx-app-generator/stx-app-generator/tests/unit/resources/adminer-0.2.1.tgz
chartGroup: chartGroup-adminer
#################################################
## App Metadata Configuration

BIN
stx-app-generator/stx-app-generator/tests/unit/resources/tempo-vulture-0.4.1.tgz Normal file
View File

Binary file not shown.

54
stx-app-generator/stx-app-generator/tests/unit/test_application_class.py
View File

@ -1,33 +1,24 @@
"""This testfile tests the Application class"""
import git
import os
import shutil
import tarfile
from app_gen_tool.common import get_chart_from_git
from app_gen_tool.fluxcd import FluxCD
from app_gen_tool.generator import parse_yaml, create_app_directories, check_input_file
def _extract(tar_url, extract_path='.'):
"""Extracts content from a .tar package stored on the web"""
tar = tarfile.open(tar_url, 'r')
for item in tar:
tar.extract(item, extract_path)
if item.name.find(".tgz") != -1 or item.name.find(".tar") != -1:
_extract(item.name, "./" + item.name[:item.name.rfind('/')])
class TestFluxCDAppGen:
"""Class to run unit tests on app-gen"""
def setup_class(cls):
current_file = os.path.abspath(__file__)
current_folder = os.path.dirname(current_file)
resource_folder = os.path.join(current_folder, 'resources')
input_file = os.path.join(resource_folder, 'app-test.yaml')
test_helm_chart = os.path.join(resource_folder, 'adminer-0.2.1.tgz')
cls.current_file = os.path.abspath(__file__)
cls.current_folder = os.path.dirname(cls.current_file)
cls.resource_folder = os.path.join(cls.current_folder, 'resources')
input_file = os.path.join(cls.resource_folder, 'app-test.yaml')
cls.OUTPUT_FOLDER = os.path.join(current_folder, "TEST_OUTPUT")
cls.helm_chart_folder = os.path.join(resource_folder, 'adminer')
cls.OUTPUT_FOLDER = os.path.join(cls.current_folder, "TEST_OUTPUT")
cls.helm_chart_folder = os.path.join(cls.resource_folder, 'adminer')
cls.app_data = parse_yaml(input_file)
try:
@ -42,8 +33,7 @@ class TestFluxCDAppGen:
if not os.path.exists(cls.OUTPUT_FOLDER):
os.makedirs(cls.OUTPUT_FOLDER, exist_ok=True)
if not os.path.exists(cls.helm_chart_folder):
_extract(test_helm_chart, cls.helm_chart_folder)
def teardown_class(cls):
@ -75,3 +65,27 @@ class TestFluxCDAppGen:
file_exists = os.path.exists(output_file)
assert file_exists
# assert True
def test_clone_helm_chart_from_git(self):
"""Test cloning helm chart from a git url"""
helm_chart_git_url = "https://github.com/cetic/helm-adminer.git"
tmp_folder = os.path.join(self.current_folder, "tmp-git-test")
try:
if not os.path.exists(tmp_folder):
os.makedirs(tmp_folder, exist_ok=False)
git.Repo.clone_from(helm_chart_git_url, tmp_folder)
git_name = "helm-adminer"
new_path = os.path.join(tmp_folder, git_name)
get_chart_from_git(new_path, tmp_folder, helm_chart_git_url, git_name)
assert os.path.exists(new_path)
except git.exc.GitError:
print(f"git repository {helm_chart_git_url} does not exist or is not public!")
assert False
except Exception as e:
print(f"Exception error {e} occurred!")
assert False
finally:
if os.path.exists(tmp_folder):
shutil.rmtree(tmp_folder)
assert not os.path.exists(tmp_folder)

17
stx-app-generator/stx-app-generator/tests/unit/test_camel_case.py
View File

@ -1,17 +0,0 @@
from app_gen_tool.common import to_camel_case
class TestCameCase:
def setup_class(cls):
pass
def teardown_class(cls):
pass
def test_camel_case(self):
input_string = "ThisIsAString"
output_string = to_camel_case(input_string)
assert output_string == input_string.lower()

59
stx-app-generator/stx-app-generator/tests/unit/test_common_functions.py Normal file
View File

@ -0,0 +1,59 @@
"""This testfile tests the common functions used across the repository"""
import os
import shutil
from app_gen_tool.common import to_camel_case
from app_gen_tool.common import extract, transform_rel_path_into_abs_path
class CommonFunctions:
def __init__(self):
self.current_file = os.path.abspath(__file__)
self.current_folder = os.path.dirname(self.current_file)
self.resource_folder = os.path.join(self.current_folder, 'resources')
def setup_class(cls):
pass
def teardown_class(cls):
pass
def test_camel_case(self):
input_string = "ThisIsAString"
output_string = to_camel_case(input_string)
assert output_string == input_string.lower()
def test_tar_extraction(self):
"""Test tar extraction function"""
tmp_folder = os.path.join('/tmp', "tmp-tar-test")
try:
if not os.path.exists(tmp_folder):
os.makedirs(tmp_folder, exist_ok=False)
app_name = 'tempo-vulture'
tarfile = 'tempo-vulture-0.4.1.tgz'
tarfile_path = os.path.join(self.resource_folder, tarfile)
extract(tarfile_path, tmp_folder)
assert os.path.exists(os.path.join(tmp_folder, app_name))
except Exception as e:
print(f"Exception error {e} occurred!")
assert False
finally:
if os.path.exists(tmp_folder):
shutil.rmtree(tmp_folder)
assert not os.path.exists(tmp_folder)
def test_transform_path_function(self):
rel_path_1 = '../../../../example/poc-starlingx-messages'
rel_path_2 = '../../../../example/tempo-vulture/tempo-vulture-0.4.1.tgz'
rel_path_3 = './resources/adminer-0.2.1.tgz'
tar_name_1 = '/tempo-vulture-0.4.1.tgz'
tar_name_2 = '/adminer-0.2.1.tgz'
abs_path_1 = transform_rel_path_into_abs_path(rel_path_1)
abs_path_2 = transform_rel_path_into_abs_path(rel_path_2)
abs_path_3 = transform_rel_path_into_abs_path(rel_path_3)
abs_path_2 = abs_path_2.replace(tar_name_1, '', 1)
abs_path_3 = abs_path_3.replace(tar_name_2, '', 1)
if os.path.exists(abs_path_1) and os.path.exists(abs_path_2) and os.path.exists(abs_path_3):
assert True
assert False

4
test-requirements.txt
View File

@ -6,4 +6,6 @@ yamllint<1.26.1;python_version>="3.0" # GPLv2
pylint
bandit
pytest
coverage
coverage
gitpython
wheel