From 4792d66618b46bf129b5325cf18e3525ca4ec138 Mon Sep 17 00:00:00 2001 From: Don Penney Date: Thu, 6 Jun 2019 15:53:30 -0400 Subject: [PATCH] Enhance filtering of software groups for install Software groups are generated using the platform_comps.py utility to declare what software is installed on the various node types, generating a comps.xml for the base software repo in the ISO (and on the controller) with package groups for each node type. The "filter_out_from_X" files provide a list of packages to exclude from the corresponding software group. This update enhances the filtering by using the package list from the image.inc files, rather than the full set of packages included in the ISO. The latter, which is the existing implementation, would include all packages pulled in to meet dependency requirements in addition to those specified in image.inc files. As a result, we would also need to include these dependencies in the filter files to ensure they are not installed on a given node. By using just the package list from the image.inc files, the software groups will only include the requested packages. Adding a package to the filter file would then effectively also filter its dependencies. For example, if package A is in an image.inc file and requires package B, and we include A in the filter_out_from_storage list, the existing implementation would still include B in the storage software group and B would be installed on the node. With this updated implementation, B would not be included in the software groups at all, and only installed on nodes that also install A. Additionally, this update also cleans up filter references from centos-ks-gen.pl which were obsolete and should have been removed previously. This update also adds starlingx-dashboard to the filter lists for storage and worker nodes. As an example, this update has the net effect of reducing the number of packages installed on a worker node from the current total of 1060 to 932 packages, by trimming the unneeded packages pulled in from dependencies of filtered packages. Change-Id: I1c1c3fabf53c4a1dd085f135c7443e09f96e5906 Story: 2004764 Task: 33602 Signed-off-by: Don Penney --- bsp-files/centos-ks-gen.pl | 44 ++++++++------------- bsp-files/filter_out_from_storage | 1 + bsp-files/filter_out_from_worker | 1 + bsp-files/filter_out_from_worker_lowlatency | 1 + bsp-files/platform_comps.py | 29 ++++++++------ 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/bsp-files/centos-ks-gen.pl b/bsp-files/centos-ks-gen.pl index 6109ef5d..b4efc591 100755 --- a/bsp-files/centos-ks-gen.pl +++ b/bsp-files/centos-ks-gen.pl @@ -37,7 +37,7 @@ system("mkdir -p ${output_dir}"); # Write USB image files write_config_file("controller", - "${output_dir}/controller_ks.cfg", "filter_out_from_controller", + "${output_dir}/controller_ks.cfg", "pre_common_head.cfg", "pre_pkglist.cfg", "pre_disk_setup_common.cfg", @@ -48,7 +48,7 @@ write_config_file("controller", "post_lvm_pv_on_rootfs.cfg", "post_usb_controller.cfg"); write_config_file("controller-worker", - "${output_dir}/smallsystem_ks.cfg", "filter_out_from_smallsystem", + "${output_dir}/smallsystem_ks.cfg", "pre_common_head.cfg", "pre_pkglist.cfg", "pre_disk_setup_common.cfg", @@ -60,7 +60,7 @@ write_config_file("controller-worker", "post_system_aio.cfg", "post_usb_controller.cfg"); write_config_file("controller-worker-lowlatency", - "${output_dir}/smallsystem_lowlatency_ks.cfg", "filter_out_from_smallsystem_lowlatency", + "${output_dir}/smallsystem_lowlatency_ks.cfg", "pre_common_head.cfg", "pre_pkglist_lowlatency.cfg", "pre_disk_setup_common.cfg", @@ -76,7 +76,7 @@ system("mkdir -p ${pxeboot_output_dir}"); # Write PXE boot files write_config_file("controller", - "${pxeboot_output_dir}/pxeboot_controller.cfg", "filter_out_from_controller", + "${pxeboot_output_dir}/pxeboot_controller.cfg", "pre_common_head.cfg", "pre_pkglist.cfg", "pre_disk_setup_common.cfg", @@ -87,7 +87,7 @@ write_config_file("controller", "post_lvm_pv_on_rootfs.cfg", "post_pxeboot_controller.cfg"); write_config_file("controller-worker", - "${pxeboot_output_dir}/pxeboot_smallsystem.cfg", "filter_out_from_smallsystem", + "${pxeboot_output_dir}/pxeboot_smallsystem.cfg", "pre_common_head.cfg", "pre_pkglist.cfg", "pre_disk_setup_common.cfg", @@ -99,7 +99,7 @@ write_config_file("controller-worker", "post_system_aio.cfg", "post_pxeboot_controller.cfg"); write_config_file("controller-worker-lowlatency", - "${pxeboot_output_dir}/pxeboot_smallsystem_lowlatency.cfg", "filter_out_from_smallsystem_lowlatency", + "${pxeboot_output_dir}/pxeboot_smallsystem_lowlatency.cfg", "pre_common_head.cfg", "pre_pkglist_lowlatency.cfg", "pre_disk_setup_common.cfg", @@ -114,7 +114,7 @@ write_config_file("controller-worker-lowlatency", # Write same net files write_config_file("controller", - "${output_dir}/net_controller_ks.cfg", "filter_out_from_controller", + "${output_dir}/net_controller_ks.cfg", "pre_common_head.cfg", "pre_net_common.cfg", "pre_pkglist.cfg", @@ -127,7 +127,7 @@ write_config_file("controller", "post_net_controller.cfg", "post_net_common.cfg"); write_config_file("controller-worker", - "${output_dir}/net_smallsystem_ks.cfg", "filter_out_from_smallsystem", + "${output_dir}/net_smallsystem_ks.cfg", "pre_common_head.cfg", "pre_net_common.cfg", "pre_pkglist.cfg", @@ -141,7 +141,7 @@ write_config_file("controller-worker", "post_net_controller.cfg", "post_net_common.cfg"); write_config_file("controller-worker-lowlatency", - "${output_dir}/net_smallsystem_lowlatency_ks.cfg", "filter_out_from_smallsystem_lowlatency", + "${output_dir}/net_smallsystem_lowlatency_ks.cfg", "pre_common_head.cfg", "pre_net_common.cfg", "pre_pkglist_lowlatency.cfg", @@ -155,7 +155,7 @@ write_config_file("controller-worker-lowlatency", "post_net_controller.cfg", "post_net_common.cfg"); write_config_file("worker", - "${output_dir}/net_worker_ks.cfg", "filter_out_from_worker", + "${output_dir}/net_worker_ks.cfg", "pre_common_head.cfg", "pre_net_common.cfg", "pre_pkglist.cfg", @@ -167,7 +167,7 @@ write_config_file("worker", "post_lvm_pv_on_rootfs.cfg", "post_net_common.cfg"); write_config_file("worker-lowlatency", - "${output_dir}/net_worker_lowlatency_ks.cfg", "filter_out_from_worker_lowlatency", + "${output_dir}/net_worker_lowlatency_ks.cfg", "pre_common_head.cfg", "pre_net_common.cfg", "pre_pkglist_lowlatency.cfg", @@ -179,7 +179,7 @@ write_config_file("worker-lowlatency", "post_lvm_pv_on_rootfs.cfg", "post_net_common.cfg"); write_config_file("storage", - "${output_dir}/net_storage_ks.cfg", "filter_out_from_storage", + "${output_dir}/net_storage_ks.cfg", "pre_common_head.cfg", "pre_net_common.cfg", "pre_pkglist.cfg", @@ -200,7 +200,7 @@ foreach $server (keys %boot_servers) $BOOT_SERVER = $boot_servers{$server}; write_config_file("controller", - "${extra_output_dir}/${server}_controller.cfg", "filter_out_from_controller", + "${extra_output_dir}/${server}_controller.cfg", "pre_common_head.cfg", "pre_pkglist.cfg", "pre_disk_setup_common.cfg", @@ -211,7 +211,7 @@ foreach $server (keys %boot_servers) "post_lvm_pv_on_rootfs.cfg", "post_yow_controller.cfg"); write_config_file("controller-worker", - "${extra_output_dir}/${server}_smallsystem.cfg", "filter_out_from_smallsystem", + "${extra_output_dir}/${server}_smallsystem.cfg", "pre_common_head.cfg", "pre_pkglist.cfg", "pre_disk_setup_common.cfg", @@ -223,7 +223,7 @@ foreach $server (keys %boot_servers) "post_system_aio.cfg", "post_yow_controller.cfg"); write_config_file("controller-worker-lowlatency", - "${extra_output_dir}/${server}_smallsystem_lowlatency.cfg", "filter_out_from_smallsystem_lowlatency", + "${extra_output_dir}/${server}_smallsystem_lowlatency.cfg", "pre_common_head.cfg", "pre_pkglist_lowlatency.cfg", "pre_disk_setup_common.cfg", @@ -241,19 +241,7 @@ exit 0; #------------------------# sub write_config_file { - my ($personality, $ksout, $filter_file, @templates) = @_; - my %filter; - if ($filter_file ne "") { - if (!(open(FILTER, "$files_dir/$filter_file"))) { - die "Could not open template $files_dir/$filter_file"; - } - while () { - chop(); - next if ($_ =~ /^#/); - $filter{$_} = 1; - } - close(FILTER); - } + my ($personality, $ksout, @templates) = @_; print "Writing: $ksout\n"; open(OUT, ">$ksout") || die "Could not write $ksout"; diff --git a/bsp-files/filter_out_from_storage b/bsp-files/filter_out_from_storage index a1fb715d..a646bd9d 100644 --- a/bsp-files/filter_out_from_storage +++ b/bsp-files/filter_out_from_storage @@ -268,6 +268,7 @@ sm-client sm-common sm-db sm-tools +starlingx-dashboard storage-topology nfv nfv-common diff --git a/bsp-files/filter_out_from_worker b/bsp-files/filter_out_from_worker index aaac4ebf..cf4b3750 100644 --- a/bsp-files/filter_out_from_worker +++ b/bsp-files/filter_out_from_worker @@ -145,6 +145,7 @@ sm-client sm-common sm-db sm-tools +starlingx-dashboard storageconfig storage-topology task-cloud-controller diff --git a/bsp-files/filter_out_from_worker_lowlatency b/bsp-files/filter_out_from_worker_lowlatency index 9fb60c38..d0efbc0e 100644 --- a/bsp-files/filter_out_from_worker_lowlatency +++ b/bsp-files/filter_out_from_worker_lowlatency @@ -148,6 +148,7 @@ sm-client sm-common sm-db sm-tools +starlingx-dashboard storageconfig storage-topology task-cloud-controller diff --git a/bsp-files/platform_comps.py b/bsp-files/platform_comps.py index 3b0d0361..8bfa34f5 100644 --- a/bsp-files/platform_comps.py +++ b/bsp-files/platform_comps.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ -Copyright (c) 2018 Wind River Systems, Inc. +Copyright (c) 2018-2019 Wind River Systems, Inc. SPDX-License-Identifier: Apache-2.0 @@ -15,7 +15,7 @@ import xml.etree.ElementTree as ElementTree def usage(): - print("Usage: %s --groups --pkgdir " + print("Usage: %s --groups --pkglist " % os.path.basename(sys.argv[0])) exit(1) @@ -88,29 +88,36 @@ def main(): try: opts, remainder = getopt.getopt(sys.argv[1:], '', - ['pkgdir=', - 'groups=']) + ['pkgdir=', # Deprecated + 'groups=', + 'pkglist=']) except getopt.GetoptError: usage() - pkgdir = None groups_file = None + pkglist = [] # Filters are colocated with this script filter_dir = os.path.dirname(sys.argv[0]) for opt, arg in opts: - if opt == "--pkgdir": - pkgdir = arg - elif opt == "--groups": + if opt == "--groups": groups_file = arg + elif opt == "--pkglist": + pkglist.append(arg) - if pkgdir is None or groups_file is None: + if groups_file is None: usage() + if len(pkglist) == 0: + # Use default files + pkglist.append(os.path.join(os.environ['MY_REPO'], + 'build-tools/build_iso/minimal_rpm_list.txt')) + pkglist.append(os.path.join(os.environ['MY_WORKSPACE'], + 'std/image.inc')) + # Get the pkglist - cmd = "find %s -name '*.rpm' \ - | xargs rpm -qp --queryformat '%%{NAME}\n'" % pkgdir + cmd = "sed 's/#.*//' %s" % ' '.join(pkglist) rpmlist = subprocess.check_output(cmd, shell=True).split() tree = ElementTree.parse(groups_file)