starlingx/metal
Code Issues Proposed changes

Enhance filtering of software groups for install

Browse Source 
Software groups are generated using the platform_comps.py
utility to declare what software is installed on the
various node types, generating a comps.xml for the base
software repo in the ISO (and on the controller) with
package groups for each node type. The "filter_out_from_X"
files provide a list of packages to exclude from the
corresponding software group.

This update enhances the filtering by using the package
list from the image.inc files, rather than the full set of
packages included in the ISO. The latter, which is the existing
implementation, would include all packages pulled in to meet
dependency requirements in addition to those specified in
image.inc files. As a result, we would also need to include
these dependencies in the filter files to ensure they are not
installed on a given node.

By using just the package list from the image.inc files, the
software groups will only include the requested packages. Adding
a package to the filter file would then effectively also filter
its dependencies.

For example, if package A is in an image.inc file and requires
package B, and we include A in the filter_out_from_storage list,
the existing implementation would still include B in the storage
software group and B would be installed on the node. With this
updated implementation, B would not be included in the software
groups at all, and only installed on nodes that also install A.

Additionally, this update also cleans up filter references from
centos-ks-gen.pl which were obsolete and should have been removed
previously.

This update also adds starlingx-dashboard to the filter lists for
storage and worker nodes.

As an example, this update has the net effect of reducing the
number of packages installed on a worker node from the current
total of 1060 to 932 packages, by trimming the unneeded packages
pulled in from dependencies of filtered packages.

Change-Id: I1c1c3fabf53c4a1dd085f135c7443e09f96e5906
Story: 2004764
Task: 33602
Signed-off-by: Don Penney <don.penney@windriver.com>
This commit is contained in:
Don Penney 2019-06-06 15:53:30 -04:00
parent 1611948e2d
commit 4792d66618
5 changed files with 37 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
bsp-files/centos-ks-gen.pl
View File

@ -37,7 +37,7 @@ system("mkdir -p ${output_dir}");
# Write USB image files
write_config_file("controller",
"${output_dir}/controller_ks.cfg", "filter_out_from_controller",
"${output_dir}/controller_ks.cfg",
"pre_common_head.cfg",
"pre_pkglist.cfg",
"pre_disk_setup_common.cfg",
@ -48,7 +48,7 @@ write_config_file("controller",
"post_lvm_pv_on_rootfs.cfg",
"post_usb_controller.cfg");
write_config_file("controller-worker",
"${output_dir}/smallsystem_ks.cfg", "filter_out_from_smallsystem",
"${output_dir}/smallsystem_ks.cfg",
"pre_common_head.cfg",
"pre_pkglist.cfg",
"pre_disk_setup_common.cfg",
@ -60,7 +60,7 @@ write_config_file("controller-worker",
"post_system_aio.cfg",
"post_usb_controller.cfg");
write_config_file("controller-worker-lowlatency",
"${output_dir}/smallsystem_lowlatency_ks.cfg", "filter_out_from_smallsystem_lowlatency",
"${output_dir}/smallsystem_lowlatency_ks.cfg",
"pre_common_head.cfg",
"pre_pkglist_lowlatency.cfg",
"pre_disk_setup_common.cfg",
@ -76,7 +76,7 @@ system("mkdir -p ${pxeboot_output_dir}");
# Write PXE boot files
write_config_file("controller",
"${pxeboot_output_dir}/pxeboot_controller.cfg", "filter_out_from_controller",
"${pxeboot_output_dir}/pxeboot_controller.cfg",
"pre_common_head.cfg",
"pre_pkglist.cfg",
"pre_disk_setup_common.cfg",
@ -87,7 +87,7 @@ write_config_file("controller",
"post_lvm_pv_on_rootfs.cfg",
"post_pxeboot_controller.cfg");
write_config_file("controller-worker",
"${pxeboot_output_dir}/pxeboot_smallsystem.cfg", "filter_out_from_smallsystem",
"${pxeboot_output_dir}/pxeboot_smallsystem.cfg",
"pre_common_head.cfg",
"pre_pkglist.cfg",
"pre_disk_setup_common.cfg",
@ -99,7 +99,7 @@ write_config_file("controller-worker",
"post_system_aio.cfg",
"post_pxeboot_controller.cfg");
write_config_file("controller-worker-lowlatency",
"${pxeboot_output_dir}/pxeboot_smallsystem_lowlatency.cfg", "filter_out_from_smallsystem_lowlatency",
"${pxeboot_output_dir}/pxeboot_smallsystem_lowlatency.cfg",
"pre_common_head.cfg",
"pre_pkglist_lowlatency.cfg",
"pre_disk_setup_common.cfg",
@ -114,7 +114,7 @@ write_config_file("controller-worker-lowlatency",
# Write same net files
write_config_file("controller",
"${output_dir}/net_controller_ks.cfg", "filter_out_from_controller",
"${output_dir}/net_controller_ks.cfg",
"pre_common_head.cfg",
"pre_net_common.cfg",
"pre_pkglist.cfg",
@ -127,7 +127,7 @@ write_config_file("controller",
"post_net_controller.cfg",
"post_net_common.cfg");
write_config_file("controller-worker",
"${output_dir}/net_smallsystem_ks.cfg", "filter_out_from_smallsystem",
"${output_dir}/net_smallsystem_ks.cfg",
"pre_common_head.cfg",
"pre_net_common.cfg",
"pre_pkglist.cfg",
@ -141,7 +141,7 @@ write_config_file("controller-worker",
"post_net_controller.cfg",
"post_net_common.cfg");
write_config_file("controller-worker-lowlatency",
"${output_dir}/net_smallsystem_lowlatency_ks.cfg", "filter_out_from_smallsystem_lowlatency",
"${output_dir}/net_smallsystem_lowlatency_ks.cfg",
"pre_common_head.cfg",
"pre_net_common.cfg",
"pre_pkglist_lowlatency.cfg",
@ -155,7 +155,7 @@ write_config_file("controller-worker-lowlatency",
"post_net_controller.cfg",
"post_net_common.cfg");
write_config_file("worker",
"${output_dir}/net_worker_ks.cfg", "filter_out_from_worker",
"${output_dir}/net_worker_ks.cfg",
"pre_common_head.cfg",
"pre_net_common.cfg",
"pre_pkglist.cfg",
@ -167,7 +167,7 @@ write_config_file("worker",
"post_lvm_pv_on_rootfs.cfg",
"post_net_common.cfg");
write_config_file("worker-lowlatency",
"${output_dir}/net_worker_lowlatency_ks.cfg", "filter_out_from_worker_lowlatency",
"${output_dir}/net_worker_lowlatency_ks.cfg",
"pre_common_head.cfg",
"pre_net_common.cfg",
"pre_pkglist_lowlatency.cfg",
@ -179,7 +179,7 @@ write_config_file("worker-lowlatency",
"post_lvm_pv_on_rootfs.cfg",
"post_net_common.cfg");
write_config_file("storage",
"${output_dir}/net_storage_ks.cfg", "filter_out_from_storage",
"${output_dir}/net_storage_ks.cfg",
"pre_common_head.cfg",
"pre_net_common.cfg",
"pre_pkglist.cfg",
@ -200,7 +200,7 @@ foreach $server (keys %boot_servers)
$BOOT_SERVER = $boot_servers{$server};
write_config_file("controller",
"${extra_output_dir}/${server}_controller.cfg", "filter_out_from_controller",
"${extra_output_dir}/${server}_controller.cfg",
"pre_common_head.cfg",
"pre_pkglist.cfg",
"pre_disk_setup_common.cfg",
@ -211,7 +211,7 @@ foreach $server (keys %boot_servers)
"post_lvm_pv_on_rootfs.cfg",
"post_yow_controller.cfg");
write_config_file("controller-worker",
"${extra_output_dir}/${server}_smallsystem.cfg", "filter_out_from_smallsystem",
"${extra_output_dir}/${server}_smallsystem.cfg",
"pre_common_head.cfg",
"pre_pkglist.cfg",
"pre_disk_setup_common.cfg",
@ -223,7 +223,7 @@ foreach $server (keys %boot_servers)
"post_system_aio.cfg",
"post_yow_controller.cfg");
write_config_file("controller-worker-lowlatency",
"${extra_output_dir}/${server}_smallsystem_lowlatency.cfg", "filter_out_from_smallsystem_lowlatency",
"${extra_output_dir}/${server}_smallsystem_lowlatency.cfg",
"pre_common_head.cfg",
"pre_pkglist_lowlatency.cfg",
"pre_disk_setup_common.cfg",
@ -241,19 +241,7 @@ exit 0;
#------------------------#
sub write_config_file {
my ($personality, $ksout, $filter_file, @templates) = @_;
my %filter;
if ($filter_file ne "") {
if (!(open(FILTER, "$files_dir/$filter_file"))) {
die "Could not open template $files_dir/$filter_file";
}
while (<FILTER>) {
chop();
next if ($_ =~ /^#/);
$filter{$_} = 1;
}
close(FILTER);
}
my ($personality, $ksout, @templates) = @_;
print "Writing: $ksout\n";
open(OUT, ">$ksout") || die "Could not write $ksout";

1
bsp-files/filter_out_from_storage
View File

@ -268,6 +268,7 @@ sm-client
sm-common
sm-db
sm-tools
starlingx-dashboard
storage-topology
nfv
nfv-common

1
bsp-files/filter_out_from_worker
View File

@ -145,6 +145,7 @@ sm-client
sm-common
sm-db
sm-tools
starlingx-dashboard
storageconfig
storage-topology
task-cloud-controller

1
bsp-files/filter_out_from_worker_lowlatency
View File

@ -148,6 +148,7 @@ sm-client
sm-common
sm-db
sm-tools
starlingx-dashboard
storageconfig
storage-topology
task-cloud-controller

29
bsp-files/platform_comps.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
"""
Copyright (c) 2018 Wind River Systems, Inc.
Copyright (c) 2018-2019 Wind River Systems, Inc.
SPDX-License-Identifier: Apache-2.0
@ -15,7 +15,7 @@ import xml.etree.ElementTree as ElementTree
def usage():
print("Usage: %s --groups <groups.xml> --pkgdir <pkgdir>"
print("Usage: %s --groups <groups.xml> --pkglist <pkglist>"
% os.path.basename(sys.argv[0]))
exit(1)
@ -88,29 +88,36 @@ def main():
try:
opts, remainder = getopt.getopt(sys.argv[1:],
'',
['pkgdir=',
'groups='])
['pkgdir=', # Deprecated
'groups=',
'pkglist='])
except getopt.GetoptError:
usage()
pkgdir = None
groups_file = None
pkglist = []
# Filters are colocated with this script
filter_dir = os.path.dirname(sys.argv[0])
for opt, arg in opts:
if opt == "--pkgdir":
pkgdir = arg
elif opt == "--groups":
if opt == "--groups":
groups_file = arg
elif opt == "--pkglist":
pkglist.append(arg)
if pkgdir is None or groups_file is None:
if groups_file is None:
usage()
if len(pkglist) == 0:
# Use default files
pkglist.append(os.path.join(os.environ['MY_REPO'],
'build-tools/build_iso/minimal_rpm_list.txt'))
pkglist.append(os.path.join(os.environ['MY_WORKSPACE'],
'std/image.inc'))
# Get the pkglist
cmd = "find %s -name '*.rpm' \
| xargs rpm -qp --queryformat '%%{NAME}\n'" % pkgdir
cmd = "sed 's/#.*//' %s" % ' '.join(pkglist)
rpmlist = subprocess.check_output(cmd, shell=True).split()
tree = ElementTree.parse(groups_file)