starlingx/update
Code Issues Proposed changes

Merge "Support uploading pre USM load in DC"

Browse Source
This commit is contained in:
Zuul 2024-09-09 20:39:49 +00:00 committed by Gerrit Code Review
commit 4d497a4fbb
6 changed files with 354 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
software/scripts/usm_load_import
View File

@ -25,6 +25,8 @@ import sys
import upgrade_utils
AVAILABLE_DIR = "/opt/software/metadata/available"
UNAVAILABLE_DIR = "/opt/software/metadata/unavailable"
COMMITTED_DIR = "/opt/software/metadata/committed"
FEED_OSTREE_BASE_DIR = "/var/www/pages/feed"
RELEASE_GA_NAME = "starlingx-%s"
SOFTWARE_STORAGE_DIR = "/opt/software"
@ -183,6 +185,13 @@ def load_import(from_release, to_release, iso_mount_dir):
LOG.info("Removed %s", to_feed_dir)
raise
def move_metadata_file_to_available_dir(to_release, iso_mount_dir):
"""
Move release metadata file to /opt/software/metadata/available
:param to_release: release version
:param iso_mount_dir: iso mount dir
"""
try:
# Copy metadata.xml to /opt/software/metadata/available
os.makedirs(AVAILABLE_DIR, exist_ok=True)
@ -203,6 +212,56 @@ def load_import(from_release, to_release, iso_mount_dir):
raise
def generate_metadata_file_in_unavailable_dir(to_release):
"""
Generate release metadata file in /opt/software/metadata/unavailable
This is only for 22.12 pre USM iso load import
:param to_release: release version
"""
try:
# Copy metadata.xml to /opt/software/metadata/unavailable
os.makedirs(UNAVAILABLE_DIR, exist_ok=True)
# TODO(jli14): release name should be dynamically generated based on the branch.
metadata_name = f"{RELEASE_GA_NAME % to_release}-metadata.xml"
LOG.info("metadata name: %s", metadata_name)
# Generate metadata.xml
import xml.etree.ElementTree as ET
from xml.dom import minidom
root = ET.Element('patch')
id_elem = ET.SubElement(root, "id")
id_elem.text = RELEASE_GA_NAME % to_release
xml_str = ET.tostring(root, encoding='unicode')
pretty_xml = minidom.parseString(xml_str).toprettyxml(indent=" ")
pretty_xml = '\n'.join([line for line in pretty_xml.split('\n') if line.strip()])
# Write to file
abs_path_metadata_filename = os.path.join(UNAVAILABLE_DIR, metadata_name)
with open(abs_path_metadata_filename, "w") as file:
file.write(pretty_xml)
except Exception:
LOG.exception("Failed to copy the release %s metadata file to %s" %
(to_release, UNAVAILABLE_DIR))
raise
def copy_patch_metadata_files_to_committed(iso_mount_dir):
"""
Copy patch metadata files to /opt/software/metadata/committed
:param iso_mount_dir: iso mount dir
"""
committed_patch_dir = os.path.join(iso_mount_dir, 'patches')
try:
shutil.copytree(committed_patch_dir, COMMITTED_DIR, dirs_exist_ok=True)
LOG.info("Copied patch metadata file to %s", COMMITTED_DIR)
except shutil.Error:
LOG.exception("Failed to copy patch metadata file(s) to %s" %
COMMITTED_DIR)
raise
def main():
parser = argparse.ArgumentParser(
description="Import files from uploaded iso image to controller.",
@ -226,11 +285,26 @@ def main():
help="The mounted iso image directory.",
)
parser.add_argument(
"--is-usm-iso",
required=False,
help="True if the iso supports USM upgrade.",
default=True
)
args = parser.parse_args()
try:
LOG.info("Load import from %s to %s started", args.from_release, args.to_release)
load_import(args.from_release, args.to_release, args.iso_dir)
if args.is_usm_iso == "True":
move_metadata_file_to_available_dir(args.to_release, args.iso_dir)
else:
# pre USM iso needs to generate metadata file
generate_metadata_file_in_unavailable_dir(args.to_release)
copy_patch_metadata_files_to_committed(args.iso_dir)
except Exception as e:
LOG.exception(e)
return 1

1
software/software/constants.py
View File

@ -38,6 +38,7 @@ DEPLOY_PRECHECK_SCRIPT = "deploy-precheck"
UPGRADE_UTILS_SCRIPT = "upgrade_utils.py"
DEPLOY_START_SCRIPT = "software-deploy-start"
DEPLOY_CLEANUP_SCRIPT = "deploy-cleanup"
USM_LOAD_IMPORT_SCRIPT = "usm_load_import"
SEMANTICS_DIR = "%s/semantics" % SOFTWARE_STORAGE_DIR

120
software/software/software_controller.py
View File

@ -65,6 +65,7 @@ from software.software_functions import configure_logging
from software.software_functions import mount_iso_load
from software.software_functions import unmount_iso_load
from software.software_functions import read_upgrade_support_versions
from software.software_functions import get_to_release_from_metadata_file
from software.software_functions import BasePackageData
from software.software_functions import PatchFile
from software.software_functions import package_dir
@ -1343,49 +1344,98 @@ class PatchController(PatchService):
local_error = ""
release_meta_info = {}
try:
# Copy iso /upgrades/software-deploy/ to /opt/software/rel-<rel>/bin/
to_release_bin_dir = os.path.join(
constants.SOFTWARE_STORAGE_DIR, ("rel-%s" % to_release), "bin")
if os.path.exists(to_release_bin_dir):
shutil.rmtree(to_release_bin_dir)
shutil.copytree(os.path.join(iso_mount_dir, "upgrades",
constants.SOFTWARE_DEPLOY_FOLDER), to_release_bin_dir)
def run_script_command(cmd):
LOG.info("Running load import command: %s", " ".join(cmd))
result = subprocess.run(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, check=True, text=True)
return (result.stdout, None) if result.returncode == 0 else (None, result.stdout)
# Run usm_load_import script
import_script = os.path.join(to_release_bin_dir, 'usm_load_import')
load_import_cmd = [import_script,
# Check if usm_load_import script exists in the iso
has_usm_load_import_script = os.path.isfile(os.path.join(
iso_mount_dir, 'upgrades', 'software-deploy', constants.USM_LOAD_IMPORT_SCRIPT))
if has_usm_load_import_script:
# usm_load_import script is found. This iso supports upgrade from USM
try:
# Copy iso /upgrades/software-deploy/ to /opt/software/rel-<rel>/bin/
to_release_bin_dir = os.path.join(
constants.SOFTWARE_STORAGE_DIR, ("rel-%s" % to_release), "bin")
if os.path.exists(to_release_bin_dir):
shutil.rmtree(to_release_bin_dir)
shutil.copytree(os.path.join(iso_mount_dir, "upgrades",
constants.SOFTWARE_DEPLOY_FOLDER), to_release_bin_dir)
# Run usm_load_import script
import_script = os.path.join(to_release_bin_dir, constants.USM_LOAD_IMPORT_SCRIPT)
load_import_cmd = [
str(import_script),
f"--from-release={from_release}",
f"--to-release={to_release}",
f"--iso-dir={iso_mount_dir}"
]
load_import_info, load_import_error = run_script_command(load_import_cmd)
local_info += load_import_info or ""
local_error += load_import_error or ""
# Copy metadata.xml to /opt/software/rel-<rel>/
to_file = os.path.join(constants.SOFTWARE_STORAGE_DIR,
("rel-%s" % to_release), "metadata.xml")
metadata_file = os.path.join(iso_mount_dir, "upgrades", "metadata.xml")
shutil.copyfile(metadata_file, to_file)
# Update the release metadata
# metadata files have been copied over to the metadata/available directory
reload_release_data()
LOG.info("Updated release metadata for %s", to_release)
release_meta_info = self.get_release_meta_info(
to_release, iso_mount_dir, upgrade_files)
return local_info, local_warning, local_error, release_meta_info
except Exception as e:
LOG.exception("Error occurred while running load import: %s", str(e))
raise
# At this step, usm_load_import script is not found in the iso
# Therefore, we run the local usm_load_import script which supports importing the N-1 iso
# that doesn't support USM feature.
# This is the special case where *only* DC system controller can import this iso
# TODO(ShawnLi): remove the code below when this special case is not supported
try:
local_import_script = os.path.join(
"/usr/sbin/software-deploy/", constants.USM_LOAD_IMPORT_SCRIPT)
load_import_cmd = [local_import_script,
"--from-release=%s" % from_release,
"--to-release=%s" % to_release,
"--iso-dir=%s" % iso_mount_dir]
LOG.info("Running load import command: %s", " ".join(load_import_cmd))
load_import_return = subprocess.run(load_import_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=True,
text=True)
if load_import_return.returncode != 0:
local_error += load_import_return.stdout
else:
local_info += load_import_return.stdout
"--iso-dir=%s" % iso_mount_dir,
"--is-usm-iso=False"]
# Copy metadata.xml to /opt/software/rel-<rel>/
to_file = os.path.join(constants.SOFTWARE_STORAGE_DIR,
("rel-%s" % to_release), "metadata.xml")
metadata_file = os.path.join(iso_mount_dir, "upgrades", "metadata.xml")
shutil.copyfile(metadata_file, to_file)
load_import_info, load_import_error = run_script_command(load_import_cmd)
local_info += load_import_info or ""
local_error += load_import_error or ""
# Update the release metadata
# metadata files have been copied over to the metadata/available directory
reload_release_data()
LOG.info("Updated release metadata for %s", to_release)
release_meta_info = self.get_release_meta_info(to_release, iso_mount_dir, upgrade_files)
release_meta_info = {
os.path.basename(upgrade_files[constants.ISO_EXTENSION]): {
"id": constants.RELEASE_GA_NAME % to_release,
"sw_release": to_release,
},
os.path.basename(upgrade_files[constants.SIG_EXTENSION]): {
"id": None,
"sw_release": None,
}
}
return local_info, local_warning, local_error, release_meta_info
except Exception as e:
LOG.exception("Error occurred while running load import: %s", str(e))
LOG.exception("Error occurred while running local load import script: %s", str(e))
raise
def get_release_meta_info(self, to_release, iso_mount_dir, upgrade_files) -> dict:
@ -1479,9 +1529,8 @@ class PatchController(PatchService):
try:
# Validate the N-1 release from the iso file is supported to upgrade to the current N release
_, current_upgrade_supported_versions = read_upgrade_support_versions(
"/usr/rootdirs/opt/",
do_check_to_release=False)
current_upgrade_supported_versions = read_upgrade_support_versions(
"/usr/rootdirs/opt/")
supported_versions = [v.get("version") for v in current_upgrade_supported_versions]
# to_release is N-1 release in here
@ -1492,6 +1541,8 @@ class PatchController(PatchService):
# iso validation completed
LOG.info("Starting load import from %s", upgrade_files[constants.ISO_EXTENSION])
# from_release is set to None when uploading N-1 load
return self._run_load_import(from_release, to_release, iso_mount_dir, upgrade_files)
except Exception as e:
@ -1698,7 +1749,8 @@ class PatchController(PatchService):
LOG.info("Mounted iso file %s to %s", iso, iso_mount_dir)
# Read the metadata from the iso file to get to-release and supported-from-releases
to_release, supported_from_releases = read_upgrade_support_versions(iso_mount_dir)
supported_from_releases = read_upgrade_support_versions(iso_mount_dir)
to_release = get_to_release_from_metadata_file(iso_mount_dir)
to_release_maj_ver = utils.get_major_release_version(to_release)
LOG.info("Reading metadata from iso file %s completed. \nto_release: %s", iso, to_release_maj_ver)

56
software/software/software_functions.py
View File

@ -1184,7 +1184,7 @@ def get_metadata_files(root_dir):
def get_sw_version(metadata_files):
# from a list of metadata files, find the latest sw_version (e.g 24.0.1)
unset_ver = "0.0.0"
unset_ver = constants.UNKNOWN_SOFTWARE_VERSION
rel_ver = unset_ver
for f in metadata_files:
try:
@ -1205,24 +1205,20 @@ def get_sw_version(metadata_files):
return rel_ver
def read_upgrade_support_versions(mounted_dir, do_check_to_release=True):
def read_attributes_from_metadata_file(mounted_dir):
"""
Read upgrade metadata file to get supported upgrades
versions
Get attributes from upgrade metadata xml file
:param mounted_dir: Mounted iso directory
:param do_check_to_release: True if to_release should be retrieved
:return: to_release, supported_from_releases
:return: a dict of attributes
"""
to_release = constants.UNKNOWN_SOFTWARE_VERSION
metadata_file = os.path.join(mounted_dir, "upgrades", "metadata.xml")
try:
root = ElementTree.parse(mounted_dir + "/upgrades/metadata.xml").getroot()
root = ElementTree.parse(metadata_file)
except IOError:
raise SoftwareServiceError(
"The ISO does not contain required upgrade information in upgrades/metadata.xml")
f"The ISO does not contain required upgrade information in {metadata_file}")
if do_check_to_release:
rel_metadata_files = get_metadata_files(os.path.join(mounted_dir, "upgrades"))
to_release = get_sw_version(rel_metadata_files)
to_release = root.findtext("version")
supported_from_releases = []
supported_upgrades = root.find("supported_upgrades").findall("upgrade")
@ -1231,7 +1227,41 @@ def read_upgrade_support_versions(mounted_dir, do_check_to_release=True):
"version": upgrade.findtext("version"),
"required_patch": upgrade.findtext("required_patch"),
})
return to_release, supported_from_releases
return {
"to_release": to_release,
"supported_from_releases": supported_from_releases
}
def read_upgrade_support_versions(mounted_dir):
"""
Get supported upgrades
versions
:param mounted_dir: Mounted iso directory
:param do_check_to_release: True if to_release should be retrieved
:return: supported_from_releases
"""
attributes_from_metadata = read_attributes_from_metadata_file(mounted_dir)
return attributes_from_metadata["supported_from_releases"]
def get_to_release_from_metadata_file(mounted_dir):
"""
Get to_release version
:param mounted_dir: Mounted iso directory
:return: to_release
"""
rel_metadata_files = get_metadata_files(os.path.join(mounted_dir, "upgrades"))
if len(rel_metadata_files) == 0: # This is pre-USM iso
attributes_from_metadata = read_attributes_from_metadata_file(mounted_dir)
to_release = attributes_from_metadata["to_release"]
else:
to_release = get_sw_version(rel_metadata_files)
return to_release
def create_deploy_hosts(hostname=None):

122
software/software/tests/test_software_controller.py
View File

@ -80,15 +80,14 @@ class TestSoftwareController(unittest.TestCase):
@patch('software.software_controller.SW_VERSION', '4.0.0')
@patch('software.software_controller.PatchController._run_load_import')
def test_process_inactive_upgrade_files(self,
mock_run_load_import,
mock_read_upgrade_support_versions,
mock_major_release_upload_check,
mock_init): # pylint: disable=unused-argument
mock_run_load_import,
mock_read_upgrade_support_versions,
mock_major_release_upload_check,
mock_init): # pylint: disable=unused-argument
controller = PatchController()
mock_run_load_import.return_value = "Load import successful"
mock_major_release_upload_check.return_value = True
mock_read_upgrade_support_versions.return_value = (
None, [{'version': '3.0'}, {'version': '2.0'}])
mock_read_upgrade_support_versions.return_value = [{'version': '3.0'}, {'version': '2.0'}]
from_release = None
to_release = '2.0.0'
iso_mount_dir = '/test/iso'
@ -114,8 +113,7 @@ class TestSoftwareController(unittest.TestCase):
controller = PatchController()
mock_run_load_import.return_value = "Load import successful"
mock_major_release_upload_check.return_value = True
mock_read_upgrade_support_versions.return_value = (
None, [{'version': '3.0.0'}, {'version': '2.0.0'}])
mock_read_upgrade_support_versions.return_value =[{'version': '3.0.0'}, {'version': '2.0.0'}]
from_release = None
to_release = '1.0.0'
iso_mount_dir = '/test/iso'
@ -131,6 +129,54 @@ class TestSoftwareController(unittest.TestCase):
e.message, 'ISO file release version 1.0 not supported to upgrade to 4.0.0')
@patch('software.software_controller.PatchController.__init__', return_value=None)
@patch('os.path.isfile', return_value=False)
@patch('os.path.join', return_value="/usr/sbin/software-deploy/usm_load_import")
@patch('software.software_controller.reload_release_data')
@patch('shutil.copyfile')
@patch('subprocess.run')
@patch('os.path.exists')
def test_run_load_import_success_without_usm_script(self,
mock_path_exists,
mock_subprocess_run,
mock_copyfile, # pylint: disable=unused-argument
mock_reload_release_data, # pylint: disable=unused-argument
mock_join, # pylint: disable=unused-argument
mock_isfile, # pylint: disable=unused-argument
mock_init): # pylint: disable=unused-argument
# Setup
mock_path_exists.return_value = True
mock_subprocess_run.return_value = MagicMock(returncode=0, stdout="Load import successful")
controller = PatchController()
from_release = None
to_release = "22.12"
iso_mount_dir = "/mnt/iso"
upgrade_files = {
constants.ISO_EXTENSION: "test.iso",
constants.SIG_EXTENSION: "test.sig"
}
# Call the method
local_info, local_warning, local_error, release_meta_info = controller._run_load_import( # pylint: disable=protected-access
from_release,
to_release,
iso_mount_dir,
upgrade_files)
# Assertions
self.assertEqual(local_info, "Load import successful")
self.assertEqual(local_warning, "")
self.assertEqual(local_error, "")
self.assertEqual(
release_meta_info,
{
"test.iso": {"id": "starlingx-22.12", "sw_release": "22.12"},
"test.sig": {"id": None, "sw_release": None}
}
)
@patch('software.software_controller.PatchController.__init__', return_value=None)
@patch('os.path.isfile', return_value=True)
@patch('software.software_controller.PatchController.get_release_meta_info')
@patch('software.software_controller.reload_release_data')
@patch('shutil.copyfile')
@ -138,15 +184,16 @@ class TestSoftwareController(unittest.TestCase):
@patch('shutil.copytree')
@patch('shutil.rmtree')
@patch('os.path.exists')
def test_run_load_import_success(self,
mock_path_exists,
mock_rmtree,
mock_copytree,
mock_subprocess_run,
mock_copyfile, # pylint: disable=unused-argument
mock_reload_release_data, # pylint: disable=unused-argument
mock_get_release_meta_info,
mock_init): # pylint: disable=unused-argument
def test_run_load_import_success_with_usm_script(self,
mock_path_exists,
mock_rmtree,
mock_copytree,
mock_subprocess_run,
mock_copyfile, # pylint: disable=unused-argument
mock_reload_release_data, # pylint: disable=unused-argument
mock_get_release_meta_info,
mock_isfile, # pylint: disable=unused-argument
mock_init): # pylint: disable=unused-argument
# Setup
mock_path_exists.return_value = True
mock_subprocess_run.return_value = MagicMock(returncode=0, stdout="Load import successful")
@ -177,7 +224,9 @@ class TestSoftwareController(unittest.TestCase):
mock_copytree.assert_called_once_with(
"/mnt/iso/upgrades/software-deploy", "/opt/software/rel-2.0.0/bin")
@patch('software.software_controller.PatchController.__init__', return_value=None)
@patch('os.path.isfile', return_value=True)
@patch('software.software_controller.PatchController.get_release_meta_info')
@patch('software.software_controller.reload_release_data')
@patch('shutil.copyfile')
@ -185,15 +234,16 @@ class TestSoftwareController(unittest.TestCase):
@patch('shutil.copytree')
@patch('shutil.rmtree')
@patch('os.path.exists')
def test_run_load_import_script_failure(self,
mock_path_exists,
mock_rmtree,
mock_copytree,
mock_subprocess_run,
mock_copyfile, # pylint: disable=unused-argument
mock_reload_release_data, # pylint: disable=unused-argument
mock_get_release_meta_info,
mock_init): # pylint: disable=unused-argument
def test_run_load_import_script_with_usm_script_failure(self,
mock_path_exists,
mock_rmtree,
mock_copytree,
mock_subprocess_run,
mock_copyfile, # pylint: disable=unused-argument
mock_reload_release_data, # pylint: disable=unused-argument
mock_get_release_meta_info,
mock_isfile, # pylint: disable=unused-argument
mock_init): # pylint: disable=unused-argument
# Setup
mock_path_exists.return_value = True
mock_subprocess_run.return_value = MagicMock(returncode=1, stdout="Load import failed")
@ -225,6 +275,7 @@ class TestSoftwareController(unittest.TestCase):
"/mnt/iso/upgrades/software-deploy", "/opt/software/rel-2.0.0/bin")
@patch('software.software_controller.PatchController.__init__', return_value=None)
@patch('os.path.isfile', return_value=True)
@patch('software.software_controller.PatchController.get_release_meta_info')
@patch('software.software_controller.reload_release_data')
@patch('shutil.copyfile')
@ -232,15 +283,16 @@ class TestSoftwareController(unittest.TestCase):
@patch('shutil.copytree')
@patch('shutil.rmtree')
@patch('os.path.exists')
def test_run_load_import_script_exception(self,
mock_path_exists,
mock_rmtree,
mock_copytree,
mock_subprocess_run,
mock_copyfile, # pylint: disable=unused-argument
mock_reload_release_data, # pylint: disable=unused-argument
mock_get_release_meta_info,
mock_init): # pylint: disable=unused-argument
def test_run_load_import_script_with_usm_script_exception(self,
mock_path_exists,
mock_rmtree,
mock_copytree,
mock_subprocess_run,
mock_copyfile, # pylint: disable=unused-argument
mock_reload_release_data, # pylint: disable=unused-argument
mock_get_release_meta_info,
mock_isfile, # pylint: disable=unused-argument
mock_init): # pylint: disable=unused-argument
# Setup
mock_path_exists.return_value = True
mock_subprocess_run.side_effect = Exception("Unexpected error")

63
software/software/tests/test_software_function.py
View File

@ -4,7 +4,10 @@
# Copyright (c) 2024 Wind River Systems, Inc.
#
import unittest
from unittest.mock import patch
import xml.etree.ElementTree as ET
from software.software_functions import get_to_release_from_metadata_file, read_attributes_from_metadata_file
from software.release_data import SWReleaseCollection
from software.software_functions import ReleaseData
@ -68,6 +71,7 @@ metadata2 = """<?xml version="1.0" ?>
<semantics/>
</patch>"""
expected_values = [
{
"release_id": "23.09_NRR_INSVC",
@ -191,3 +195,62 @@ class TestSoftwareFunction(unittest.TestCase):
self.assertEqual(val["pre_install"], r.pre_install)
self.assertEqual(val["commit_id"], r.commit_id)
self.assertEqual(val["checksum"], r.commit_checksum)
@patch('os.path.join')
@patch('lxml.etree.parse')
def test_read_attributes_valid_xml(self, mock_parse, mock_join):
mock_join.return_value = "/test/upgrades/metadata.xml"
# Creating a mock XML structure
root = ET.Element("root")
version_elem = ET.SubElement(root, "version")
version_elem.text = "1.0.0"
supported_upgrades_elem = ET.SubElement(root, "supported_upgrades")
upgrade_elem = ET.SubElement(supported_upgrades_elem, "upgrade")
version_elem = ET.SubElement(upgrade_elem, "version")
version_elem.text = "0.9.0"
required_patch_elem = ET.SubElement(upgrade_elem, "required_patch")
required_patch_elem.text = "patch_001"
mock_parse.return_value = root
result = read_attributes_from_metadata_file("/mocked/path")
expected_result = {
"to_release": "1.0.0",
"supported_from_releases": [
{
"version": "0.9.0",
"required_patch": "patch_001"
}
]
}
self.assertEqual(result, expected_result)
@patch('software.software_functions.get_metadata_files')
@patch('software.software_functions.read_attributes_from_metadata_file')
def test_get_to_release_from_metadata_file_without_usm(self,
mock_read_attributes,
mock_get_metadata_files):
mock_get_metadata_files.return_value = []
mock_read_attributes.return_value = {
"to_release": "1.0.0",
}
result = get_to_release_from_metadata_file('/mnt/iso')
assert result == "1.0.0"
@patch('software.software_functions.get_metadata_files')
@patch('software.software_functions.get_sw_version')
def test_get_to_release_from_metadata_file_with_usm(self,
mock_get_ver,
mock_get_metadata_files):
mock_get_metadata_files.return_value = ["/mnt/iso/metadata.xml"]
mock_get_ver.return_value = "1.0.0"
result = get_to_release_from_metadata_file('/mnt/iso')
assert result == "1.0.0"