Software upload: create pxeboot versioned dir

Details: This commit is to create pxeboot versioned directory.
It combines the lengacy functionalities of import.sh and
upgrade-start-pkg-extract scripts that setup to-release
pxeboot files.

Test Plan:

PASS: software upload and pxeboot files are in place

Task: 48999
Story: 2010676
Change-Id: I16cb7fa5e2e7bbe5784b9bb4c75ca596a54e9e8e
Signed-off-by: junfeng-li <junfeng.li@windriver.com>

software/software/constants.py

@@ -101,10 +101,7 @@ PATCH_EXTENSION = ".patch"
 SUPPORTED_UPLOAD_FILE_EXT = [ISO_EXTENSION, SIG_EXTENSION, PATCH_EXTENSION]
 SCRATCH_DIR = "/scratch"
 RELEASE_METADATA_FILE = "STX_%s_GA-metadata.xml"
-
+VAR_PXEBOOT_DIR = "/var/pxeboot"
-CONTROLLER_HOSTNAME = 'controller'
-CONTROLLER_0_HOSTNAME = '%s-0' % CONTROLLER_HOSTNAME
-CONTROLLER_1_HOSTNAME = '%s-1' % CONTROLLER_HOSTNAME
 
 # Precheck constants
 LICENSE_FILE = "/etc/platform/.license"

software/software/software_controller.py

@@ -11,6 +11,7 @@ sys.modules['osprofiler'] = None
 
 import configparser
 import gc
+import glob
 import json
 import os
 import select
@@ -58,6 +59,7 @@ from software.software_functions import ReleaseData
 from software.release_verify import verify_files
 import software.config as cfg
 import software.utils as utils
+import re
 
 import software.messages as messages
 import software.constants as constants
@@ -1004,30 +1006,8 @@ class PatchController(PatchService):
                 raise UpgradeNotSupported("Current release %s not supported to upgrade to %s"
                                           % (SW_VERSION, to_release))
 
-            # After successful validation, copy metadata.xml to /opt/software/metadata/available
+            LOG.info("Start to import iso file %s", iso_file)            
-            os.makedirs(constants.AVAILABLE_DIR, exist_ok=True)
+            threading.Thread(target=self._import_iso, args=(iso_mount_dir, to_release, release_data))
-            stx_release_metadata_file = "STX_%s_GA-metadata.xml" % to_release
-            abs_stx_release_metadata_file = os.path.join(iso_mount_dir,
-                                                         'upgrades',
-                                                         stx_release_metadata_file)
-
-            # Copy stx release metadata.xml to available metadata dir
-            shutil.copyfile(abs_stx_release_metadata_file,
-                            os.path.join(constants.AVAILABLE_DIR, stx_release_metadata_file))
-            LOG.info("Copied %s to %s", abs_stx_release_metadata_file, constants.AVAILABLE_DIR)
-
-            # Copy the iso file to /var/www/pages/feed/rel-<release>
-            os.makedirs(constants.FEED_OSTREE_BASE_DIR, exist_ok=True)
-            to_release_iso_dir = os.path.join(constants.FEED_OSTREE_BASE_DIR, ("rel-%s" % to_release))
-            shutil.copytree(iso_mount_dir, to_release_iso_dir)
-            LOG.info("Copied iso file %s to %s", iso_file, to_release_iso_dir)
-
-            # Update the release metadata
-            release_data.parse_metadata(abs_stx_release_metadata_file, state=constants.AVAILABLE)
-            LOG.info("Updated release metadata for %s", to_release)
-
-            # Unmount the iso file.
-            unmount_iso_load(iso_mount_dir)
 
         except ReleaseValidationFailure:
             msg = "Upgrade file signature verification failed"
@@ -1048,6 +1028,101 @@ class PatchController(PatchService):
             local_error += msg + "\n"
 
         return local_info, local_warning, local_error
+    
+    
+    def _import_iso(self, iso_mount_dir, to_release, release_data):
+        '''
+        Import the iso file
+        :param iso_mount_dir: iso mount directory
+        :param to_release: to release version
+        :param release_data: ReleaseData object
+        '''
+
+        try:
+            # After successful validation, copy metadata.xml to /opt/software/metadata/available
+            os.makedirs(constants.AVAILABLE_DIR, exist_ok=True)
+            stx_release_metadata_file = "STX_%s_GA-metadata.xml" % to_release
+            abs_stx_release_metadata_file = os.path.join(iso_mount_dir,
+                                                         'upgrades',
+                                                         stx_release_metadata_file)
+
+            # Copy stx release metadata.xml to available metadata dir
+            shutil.copyfile(abs_stx_release_metadata_file,
+                            os.path.join(constants.AVAILABLE_DIR, stx_release_metadata_file))
+            LOG.info("Copied %s to %s", abs_stx_release_metadata_file, constants.AVAILABLE_DIR)
+
+            # Copy the iso file to /var/www/pages/feed/rel-<release>
+            os.makedirs(constants.FEED_OSTREE_BASE_DIR, exist_ok=True)
+            to_release_feed_dir = os.path.join(constants.FEED_OSTREE_BASE_DIR, ("rel-%s" % to_release))
+            shutil.copytree(iso_mount_dir, to_release_feed_dir, dirs_exist_ok=True)
+            LOG.info("Copied iso file %s to %s", "iso_file", to_release_feed_dir)
+
+            # Copy install_uuid to /var/www/pages/feed/rel-<release>
+            from_release_feed_dir = os.path.join(constants.FEED_OSTREE_BASE_DIR, ("rel-%s" % SW_VERSION))
+            shutil.copyfile(os.path.join(from_release_feed_dir, "install_uuid"),
+                            os.path.join(to_release_feed_dir, "install_uuid"))
+
+            # Copy pxeboot-update-${SW_VERSION}.sh to from-release feed /upgrades
+            from_release_iso_upgrades_dir = os.path.join(from_release_feed_dir, "upgrades")
+            os.makedirs(from_release_iso_upgrades_dir, exist_ok=True)
+            shutil.copyfile(os.path.join("etc", "pxeboot-update-%s.sh" % SW_VERSION),
+                            os.path.join(from_release_iso_upgrades_dir, "pxeboot-update-%s.sh" % SW_VERSION))
+            
+            # Copy pxelinux.cfg.files to from-release feed /pxeboot
+            from_release_feed_pxeboot_dir = os.path.join(from_release_feed_dir, "pxeboot")
+            os.makedirs(from_release_feed_pxeboot_dir, exist_ok=True)
+            
+            # Find from-release pxelinux.cfg.files
+            pxe_dir = os.path.join(constants.VAR_PXEBOOT_DIR, "pxelinux.cfg.files")
+            from_release_pxe_files = glob.glob(os.path.join(pxe_dir, 'SW_VERSION$'))
+            for from_release_pxe_file in from_release_pxe_files:
+                if os.path.isfile(from_release_pxe_file):
+                    shutil.copyfile(from_release_pxe_file, os.path.join(from_release_feed_pxeboot_dir, 
+                                                                        os.path.basename(from_release_pxe_file)))
+            
+
+            # Converted from upgrade package extraction script
+            shutil.copyfile(os.path.join(to_release_feed_dir, "kickstart", "kickstart.cfg"),
+                            os.path.join(to_release_feed_dir, "kickstart.cfg"))
+
+            # Copy bzImage and initrd
+            bzimage_files = bzimage_files = glob.glob(os.path.join(to_release_feed_dir, 'pxeboot', 'bzImage*'))
+            for bzimage_file in bzimage_files:
+                if os.path.isfile(bzimage_file):
+                    shutil.copyfile(bzimage_file, os.path.join(constants.PXEBOOT_DIR, 
+                                                               os.path.basename(bzimage_file)))
+
+            initrd_files = glob.glob(os.path.join(to_release_feed_dir, 'pxeboot', 'initrd*'))
+            for initrd_file in initrd_files:
+                if os.path.isfile(initrd_file):
+                    shutil.copyfile(initrd_file, os.path.join(constants.PXEBOOT_DIR, 
+                                                              os.path.basename(initrd_file)))
+            
+            # Copy to_release_feed/pxelinux.cfg.files to /var/pxeboot/pxelinux.cfg.files
+            pxeboot_cfg_files = glob.glob(os.path.join(to_release_feed_dir, 'pxeboot', 'pxelinux.cfg.files',
+                                                        'SW_VERSION$'))
+            for pxeboot_cfg_file in pxeboot_cfg_files:
+                if os.path.isfile(pxeboot_cfg_file):
+                    shutil.copyfile(pxeboot_cfg_file, os.path.join(constants.PXEBOOT_DIR, 'pxelinux.cfg.files',
+                                                                    os.path.basename(pxeboot_cfg_file)))
+                    
+            # Copy pxeboot-update.sh to /etc
+            pxeboot_update_filename = "pxeboot-update-%s.sh" % to_release
+            shutil.copyfile(os.path.join(to_release_feed_dir, "upgrade", pxeboot_update_filename),
+                            os.path.join("etc", pxeboot_update_filename))
+
+            # Update the release metadata
+            release_data.parse_metadata(abs_stx_release_metadata_file, state=constants.AVAILABLE)
+            LOG.info("Updated release metadata for %s", to_release)
+
+            # Unmount the iso file.
+            unmount_iso_load(iso_mount_dir)
+
+        except Exception as e:
+            msg = "Failed to import iso"
+            LOG.exception(msg)
+            local_error += msg + "\n" 
+
 
     def _process_upload_patch_files(self, patch_files):
         '''

software/software/utils.py

@@ -159,7 +159,9 @@ def save_temp_file(file_item, temp_dir=constants.SCRATCH_DIR):
     try:
         if not os.path.exists(temp_dir):
             shutil.rmtree(temp_dir, ignore_errors=True)
-            os.makedirs(temp_dir)
+            os.makedirs(temp_dir, mode=0o755)
+            LOG.info("Created directory %s with free space %s bytes", 
+                     temp_dir, shutil.disk_usage(temp_dir).free)
     except Exception:
         raise Exception("Failed to create directory {}".format(temp_dir))
 
@@ -172,14 +174,18 @@ def save_temp_file(file_item, temp_dir=constants.SCRATCH_DIR):
             LOG.error("Not enough space to save file %s in %s \n \
                 Available %s bytes. File size %s", file_name, temp_dir, avail_space, file_size)
     except Exception:
-        LOG.exception("Failed to get file size in bytes for %s or disk space for %s", file_item, temp_dir)
+        msg = "Failed to get file size in bytes for {} or disk space for {}".format(file_item, temp_dir)
+        LOG.exception(msg)
+        raise Exception(msg)
 
     saved_file = os.path.join(temp_dir, os.path.basename(file_name))
     try:
         with open(saved_file, 'wb') as destination_file:
             destination_file.write(file_item.value)
     except Exception:
-        LOG.exception("Failed to save file %s", file_name)
+        msg = "Failed to save file {} in {}".format(file_name, temp_dir)
+        LOG.exception(msg)
+        raise Exception(msg)
 
 
 def delete_temp_file(file_name, temp_dir=constants.SCRATCH_DIR):