starlingx
/
config
Code Issues Proposed changes

Command line method to validate app metadata. 

Build a console script that allows application tarball
metadata to be validated directly via command line.

Build a console script that will allow individual apps
to execute a test via tox, where there is a single
entry point.  Whenever its updated with additional checks,
then tox will also be updated retroactively without
the need to update each app.

Fixes bug in validate_metadata_file object where a missing
app_name or app_version would be allowed.  Fixed check for
None to check for an empty string instead.

Update - Catch edge case where metadata.yaml is not found
and validate_metadata_file provides a false positive.

Usage:

For a person:
sysinv-app verify-metadata /path/to/app.tgz

For tox automation:
sysinv-app tox ./path or ./path/to/metadata.yaml

Test Plan:
PASS - Manually copy files onto AIO-SX and verify that
       the code funcitons as expected. Using an existing
       application tarball  verify that it passes checks.
PASS - Build AIO-SX ISO and verify that software functions
       as expected on a fresh install.

Story: 2010929
Task: 49194

Change-Id: I97b000489b40b6dae902e8a7b914edb9fbe5558f
Signed-off-by: Joshua Reed <joshua.reed@windriver.com>
This commit is contained in:
Joshua Reed 2023-12-04 12:47:59 -07:00 committed by Reed, Joshua
parent 33cbbaf51f
commit 5265e67cb4
4 changed files with 200 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sysinv/sysinv/debian/deb_folder/sysinv.install
View File

@ -18,6 +18,7 @@ usr/bin/cert-mon
usr/bin/kube-cert-rotation.sh
usr/bin/sysinv-agent
usr/bin/sysinv-api
usr/bin/sysinv-app
usr/bin/sysinv-conductor
usr/bin/sysinv-dbsync
usr/bin/sysinv-dnsmasq-lease-update

1
sysinv/sysinv/sysinv/setup.cfg
View File

@ -28,6 +28,7 @@ packages =
[entry_points]
console_scripts =
sysinv-api = sysinv.cmd.api:main
sysinv-app = sysinv.cmd.applications:main
sysinv-agent = sysinv.cmd.agent:main
sysinv-dbsync = sysinv.cmd.dbsync:main
sysinv-conductor = sysinv.cmd.conductor:main

59
sysinv/sysinv/sysinv/sysinv/cmd/applications.py Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env python
#
# Copyright (c) 2023 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
"""
System Inventory Application Verification & Validation Utility.
"""
import sys
from oslo_config import cfg
from sysinv.common import service
from sysinv.common.app_metadata import verify_application
from sysinv.common.app_metadata import verify_application_tarball
CONF = cfg.CONF
def verify_application_tox(path):
"""Verify an application using tox.
Args:
path: This is the direct path to the metadata.yaml file
"""
verify_application(path)
def verify_application_metadata(path):
"""Verify an application from tarball format.
Args:
path: This is a path to the app tarball.
"""
verify_application_tarball(path)
def add_action_parsers(subparsers):
parser = subparsers.add_parser('verify-metadata')
parser.set_defaults(func=verify_application_metadata)
parser.add_argument('path', nargs='?')
parser = subparsers.add_parser('tox')
parser.set_defaults(func=verify_application_tox)
parser.add_argument('path', nargs='?')
CONF.register_cli_opt(
cfg.SubCommandOpt('action',
title='actions',
help='Perform the application check operation',
handler=add_action_parsers))
def main():
service.prepare_service(sys.argv)
CONF.action.func(CONF.action.path)

143
sysinv/sysinv/sysinv/sysinv/common/app_metadata.py
View File

@ -8,8 +8,11 @@
#
import io
import glob
import os
import shutil
import six
import tempfile
import yaml
from oslo_log import log as logging
@ -22,6 +25,11 @@ from sysinv.common import utils
LOG = logging.getLogger(__name__)
def _locate_metadata_file(directory):
return glob.glob(directory + '/**/metadata.yaml', recursive=True)
def validate_metadata_file(path, metadata_file, upgrade_from_release=None):
""" Find and validate the metadata file in a given directory.
@ -303,6 +311,7 @@ def validate_metadata_file(path, metadata_file, upgrade_from_release=None):
app_version = ''
patches = []
metadata_path = os.path.join(path, metadata_file)
if os.path.isfile(metadata_path):
with io.open(metadata_path, 'r', encoding='utf-8') as f:
try:
@ -313,11 +322,15 @@ def validate_metadata_file(path, metadata_file, upgrade_from_release=None):
# metadata file does not have the key(s)
pass
if (app_name is None or
app_version is None):
# Have to check for empty string instead of None.
if app_name == '' or app_name is None:
raise exception.SysinvException(_(
"Invalid %s: app_name or/and app_version "
"is/are None." % metadata_file))
"Invalid %s: app_name is empty or None." % metadata_file)
)
if app_version == '' or app_version is None:
raise exception.SysinvException(_(
"Invalid %s: app_version is empty or None." % metadata_file)
)
behavior = validate_dict_field(doc,
constants.APP_METADATA_BEHAVIOR)
@ -427,3 +440,125 @@ def validate_metadata_file(path, metadata_file, upgrade_from_release=None):
check_release, release_patches))
return app_name, app_version, patches
def verify_application_tarball(path: str) -> None:
"""Verify metadata withing an application tarball directly.
Args:
path: str: An absolute path to application tarball.
"""
with tempfile.TemporaryDirectory() as temp_dirname:
# Copy tarball
shutil.copy(path, temp_dirname)
if not utils.extract_tarfile(temp_dirname, path):
raise Exception("Unable to extract tarball")
# If checksum file is included in the tarball, verify its contents.
if not utils.verify_checksum(temp_dirname):
raise Exception("Unable to verify app tarball checksum")
try:
name, version, _ = validate_metadata_file(
temp_dirname, constants.APP_METADATA_FILE)
if name == '' and version == '':
message = "Application Metadata file not found! Failure!"
LOG.error(message)
raise Exception(message)
else:
LOG.info(
f"Application Metadata for App: {name}, "
f"Ver: {version} succeeded!"
)
except exception.SysinvException as e:
LOG.info("Application Metadata Verification Failed!")
raise exception.SysinvException(_(
"metadata verification failed. {}".format(e)))
def verify_application_metadata_file(path: str) -> bool:
"""Verify metadata withing an that is in a repository or not in tarball.
Args:
path: str: An absolute path to application metadata.yaml or an absolute
path to the folder it resides in.
"""
is_verified = False
with tempfile.TemporaryDirectory() as temp_dirname:
# The input may be either a file, or a directory. Depending on which
# use the appropriate shutil copy function.
final_dir_name = temp_dirname
if os.path.isfile(path):
shutil.copy(path, temp_dirname)
else:
shutil.copytree(path, temp_dirname, dirs_exist_ok=True)
metadata_path_hits = _locate_metadata_file(temp_dirname)
if len(metadata_path_hits) == 0:
message = \
f"Error: Metadata file not found in directory: {path}"
LOG.error(message)
raise Exception(message)
elif len(metadata_path_hits) > 1:
message = \
"Error: Found More than One Application Metadata File! " \
"There should only be one!"
LOG.error(message)
raise Exception(message)
else:
final_dir_name = os.path.dirname(metadata_path_hits[-1])
try:
name, version, _ = validate_metadata_file(
final_dir_name, constants.APP_METADATA_FILE)
if name == '' and version == '':
message = "Application Metadata file not found! Failure!"
LOG.error(message)
is_verified = False
raise Exception(message)
else:
LOG.info(
f"Application Metadata for App: {name}, "
f"Ver: {version} succeeded!"
)
is_verified = True
except exception.SysinvException as e:
LOG.info("Application Metadata Verification Failed!")
raise exception.SysinvException(_(
"metadata verification failed. {}".format(e)))
return is_verified
def verify_application(path: str) -> bool:
"""Wrapper for all possible tests or checks. This is what Tox will use.
Whenever a new check is needed, that should be added here as another
condition.
Args:
path: str: An absolute path to application metadata.yaml or an absolute
path to the folder it resides in.
"""
is_verified = False
# For each check, add a try except so there is granularity.
# This test will exit on the first failure detected.
try:
verify_application_metadata_file(path)
is_verified = True
except exception.SysinvException as e:
LOG.info("Application Metadata Verification Failed!")
raise exception.SysinvException(_(
"metadata verification failed. {}".format(e)))
return is_verified