From 22f76f53ef75100ad838d2f81f95c46f7d75d39e Mon Sep 17 00:00:00 2001 From: junfeng-li Date: Thu, 26 Oct 2023 21:06:29 +0000 Subject: [PATCH] Software upload: create pxeboot versioned dir Details: This commit is to create pxeboot versioned directory. It combines the lengacy functionalities of import.sh and upgrade-start-pkg-extract scripts that setup to-release pxeboot files. Test Plan: PASS: software upload and pxeboot files are in place Task: 48999 Story: 2010676 Change-Id: I16cb7fa5e2e7bbe5784b9bb4c75ca596a54e9e8e Signed-off-by: junfeng-li --- software/software/constants.py | 5 +- software/software/software_controller.py | 123 ++++++++++++++++++----- software/software/utils.py | 12 ++- 3 files changed, 109 insertions(+), 31 deletions(-) diff --git a/software/software/constants.py b/software/software/constants.py index 773d7568..a051967a 100644 --- a/software/software/constants.py +++ b/software/software/constants.py @@ -101,10 +101,7 @@ PATCH_EXTENSION = ".patch" SUPPORTED_UPLOAD_FILE_EXT = [ISO_EXTENSION, SIG_EXTENSION, PATCH_EXTENSION] SCRATCH_DIR = "/scratch" RELEASE_METADATA_FILE = "STX_%s_GA-metadata.xml" - -CONTROLLER_HOSTNAME = 'controller' -CONTROLLER_0_HOSTNAME = '%s-0' % CONTROLLER_HOSTNAME -CONTROLLER_1_HOSTNAME = '%s-1' % CONTROLLER_HOSTNAME +VAR_PXEBOOT_DIR = "/var/pxeboot" # Precheck constants LICENSE_FILE = "/etc/platform/.license" diff --git a/software/software/software_controller.py b/software/software/software_controller.py index a3a11462..b76da396 100644 --- a/software/software/software_controller.py +++ b/software/software/software_controller.py @@ -11,6 +11,7 @@ sys.modules['osprofiler'] = None import configparser import gc +import glob import json import os import select @@ -58,6 +59,7 @@ from software.software_functions import ReleaseData from software.release_verify import verify_files import software.config as cfg import software.utils as utils +import re import software.messages as messages import software.constants as constants @@ -1004,30 +1006,8 @@ class PatchController(PatchService): raise UpgradeNotSupported("Current release %s not supported to upgrade to %s" % (SW_VERSION, to_release)) - # After successful validation, copy metadata.xml to /opt/software/metadata/available - os.makedirs(constants.AVAILABLE_DIR, exist_ok=True) - stx_release_metadata_file = "STX_%s_GA-metadata.xml" % to_release - abs_stx_release_metadata_file = os.path.join(iso_mount_dir, - 'upgrades', - stx_release_metadata_file) - - # Copy stx release metadata.xml to available metadata dir - shutil.copyfile(abs_stx_release_metadata_file, - os.path.join(constants.AVAILABLE_DIR, stx_release_metadata_file)) - LOG.info("Copied %s to %s", abs_stx_release_metadata_file, constants.AVAILABLE_DIR) - - # Copy the iso file to /var/www/pages/feed/rel- - os.makedirs(constants.FEED_OSTREE_BASE_DIR, exist_ok=True) - to_release_iso_dir = os.path.join(constants.FEED_OSTREE_BASE_DIR, ("rel-%s" % to_release)) - shutil.copytree(iso_mount_dir, to_release_iso_dir) - LOG.info("Copied iso file %s to %s", iso_file, to_release_iso_dir) - - # Update the release metadata - release_data.parse_metadata(abs_stx_release_metadata_file, state=constants.AVAILABLE) - LOG.info("Updated release metadata for %s", to_release) - - # Unmount the iso file. - unmount_iso_load(iso_mount_dir) + LOG.info("Start to import iso file %s", iso_file) + threading.Thread(target=self._import_iso, args=(iso_mount_dir, to_release, release_data)) except ReleaseValidationFailure: msg = "Upgrade file signature verification failed" @@ -1048,6 +1028,101 @@ class PatchController(PatchService): local_error += msg + "\n" return local_info, local_warning, local_error + + + def _import_iso(self, iso_mount_dir, to_release, release_data): + ''' + Import the iso file + :param iso_mount_dir: iso mount directory + :param to_release: to release version + :param release_data: ReleaseData object + ''' + + try: + # After successful validation, copy metadata.xml to /opt/software/metadata/available + os.makedirs(constants.AVAILABLE_DIR, exist_ok=True) + stx_release_metadata_file = "STX_%s_GA-metadata.xml" % to_release + abs_stx_release_metadata_file = os.path.join(iso_mount_dir, + 'upgrades', + stx_release_metadata_file) + + # Copy stx release metadata.xml to available metadata dir + shutil.copyfile(abs_stx_release_metadata_file, + os.path.join(constants.AVAILABLE_DIR, stx_release_metadata_file)) + LOG.info("Copied %s to %s", abs_stx_release_metadata_file, constants.AVAILABLE_DIR) + + # Copy the iso file to /var/www/pages/feed/rel- + os.makedirs(constants.FEED_OSTREE_BASE_DIR, exist_ok=True) + to_release_feed_dir = os.path.join(constants.FEED_OSTREE_BASE_DIR, ("rel-%s" % to_release)) + shutil.copytree(iso_mount_dir, to_release_feed_dir, dirs_exist_ok=True) + LOG.info("Copied iso file %s to %s", "iso_file", to_release_feed_dir) + + # Copy install_uuid to /var/www/pages/feed/rel- + from_release_feed_dir = os.path.join(constants.FEED_OSTREE_BASE_DIR, ("rel-%s" % SW_VERSION)) + shutil.copyfile(os.path.join(from_release_feed_dir, "install_uuid"), + os.path.join(to_release_feed_dir, "install_uuid")) + + # Copy pxeboot-update-${SW_VERSION}.sh to from-release feed /upgrades + from_release_iso_upgrades_dir = os.path.join(from_release_feed_dir, "upgrades") + os.makedirs(from_release_iso_upgrades_dir, exist_ok=True) + shutil.copyfile(os.path.join("etc", "pxeboot-update-%s.sh" % SW_VERSION), + os.path.join(from_release_iso_upgrades_dir, "pxeboot-update-%s.sh" % SW_VERSION)) + + # Copy pxelinux.cfg.files to from-release feed /pxeboot + from_release_feed_pxeboot_dir = os.path.join(from_release_feed_dir, "pxeboot") + os.makedirs(from_release_feed_pxeboot_dir, exist_ok=True) + + # Find from-release pxelinux.cfg.files + pxe_dir = os.path.join(constants.VAR_PXEBOOT_DIR, "pxelinux.cfg.files") + from_release_pxe_files = glob.glob(os.path.join(pxe_dir, 'SW_VERSION$')) + for from_release_pxe_file in from_release_pxe_files: + if os.path.isfile(from_release_pxe_file): + shutil.copyfile(from_release_pxe_file, os.path.join(from_release_feed_pxeboot_dir, + os.path.basename(from_release_pxe_file))) + + + # Converted from upgrade package extraction script + shutil.copyfile(os.path.join(to_release_feed_dir, "kickstart", "kickstart.cfg"), + os.path.join(to_release_feed_dir, "kickstart.cfg")) + + # Copy bzImage and initrd + bzimage_files = bzimage_files = glob.glob(os.path.join(to_release_feed_dir, 'pxeboot', 'bzImage*')) + for bzimage_file in bzimage_files: + if os.path.isfile(bzimage_file): + shutil.copyfile(bzimage_file, os.path.join(constants.PXEBOOT_DIR, + os.path.basename(bzimage_file))) + + initrd_files = glob.glob(os.path.join(to_release_feed_dir, 'pxeboot', 'initrd*')) + for initrd_file in initrd_files: + if os.path.isfile(initrd_file): + shutil.copyfile(initrd_file, os.path.join(constants.PXEBOOT_DIR, + os.path.basename(initrd_file))) + + # Copy to_release_feed/pxelinux.cfg.files to /var/pxeboot/pxelinux.cfg.files + pxeboot_cfg_files = glob.glob(os.path.join(to_release_feed_dir, 'pxeboot', 'pxelinux.cfg.files', + 'SW_VERSION$')) + for pxeboot_cfg_file in pxeboot_cfg_files: + if os.path.isfile(pxeboot_cfg_file): + shutil.copyfile(pxeboot_cfg_file, os.path.join(constants.PXEBOOT_DIR, 'pxelinux.cfg.files', + os.path.basename(pxeboot_cfg_file))) + + # Copy pxeboot-update.sh to /etc + pxeboot_update_filename = "pxeboot-update-%s.sh" % to_release + shutil.copyfile(os.path.join(to_release_feed_dir, "upgrade", pxeboot_update_filename), + os.path.join("etc", pxeboot_update_filename)) + + # Update the release metadata + release_data.parse_metadata(abs_stx_release_metadata_file, state=constants.AVAILABLE) + LOG.info("Updated release metadata for %s", to_release) + + # Unmount the iso file. + unmount_iso_load(iso_mount_dir) + + except Exception as e: + msg = "Failed to import iso" + LOG.exception(msg) + local_error += msg + "\n" + def _process_upload_patch_files(self, patch_files): ''' diff --git a/software/software/utils.py b/software/software/utils.py index 25da6177..dbc82fd1 100644 --- a/software/software/utils.py +++ b/software/software/utils.py @@ -159,7 +159,9 @@ def save_temp_file(file_item, temp_dir=constants.SCRATCH_DIR): try: if not os.path.exists(temp_dir): shutil.rmtree(temp_dir, ignore_errors=True) - os.makedirs(temp_dir) + os.makedirs(temp_dir, mode=0o755) + LOG.info("Created directory %s with free space %s bytes", + temp_dir, shutil.disk_usage(temp_dir).free) except Exception: raise Exception("Failed to create directory {}".format(temp_dir)) @@ -172,14 +174,18 @@ def save_temp_file(file_item, temp_dir=constants.SCRATCH_DIR): LOG.error("Not enough space to save file %s in %s \n \ Available %s bytes. File size %s", file_name, temp_dir, avail_space, file_size) except Exception: - LOG.exception("Failed to get file size in bytes for %s or disk space for %s", file_item, temp_dir) + msg = "Failed to get file size in bytes for {} or disk space for {}".format(file_item, temp_dir) + LOG.exception(msg) + raise Exception(msg) saved_file = os.path.join(temp_dir, os.path.basename(file_name)) try: with open(saved_file, 'wb') as destination_file: destination_file.write(file_item.value) except Exception: - LOG.exception("Failed to save file %s", file_name) + msg = "Failed to save file {} in {}".format(file_name, temp_dir) + LOG.exception(msg) + raise Exception(msg) def delete_temp_file(file_name, temp_dir=constants.SCRATCH_DIR):