From e5b865614150d3a9d67fb42c59ec4e33878f9f2f Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Mon, 17 Nov 2014 04:19:56 -0800 Subject: [PATCH] Add new package-installs system We currently support package-installs definitions which has some limitations and oddities. This new format requires only one definition which does not reside in our run-parts directories and follows a consistent naming scheme (package-installs.yaml). Change-Id: Ie51a7c4fdc15634ae8e069728e5e07cc1dc36095 --- elements/package-installs/README.rst | 36 ++++- .../package-installs/bin/package-installs-v2 | 146 ++++++++++++++++++ .../11-create-package-installs-dir | 14 ++ .../install.d/00-package-installs | 1 + .../install.d/99-package-uninstalls | 1 + .../post-install.d/00-package-installs | 1 + .../post-install.d/99-package-uninstalls | 1 + .../pre-install.d/02-package-installs | 1 + .../pre-install.d/99-package-uninstalls | 1 + tox.ini | 2 +- 10 files changed, 197 insertions(+), 7 deletions(-) create mode 100755 elements/package-installs/bin/package-installs-v2 create mode 100755 elements/package-installs/extra-data.d/11-create-package-installs-dir diff --git a/elements/package-installs/README.rst b/elements/package-installs/README.rst index 3e05a6fd4..4f5ef5fdd 100644 --- a/elements/package-installs/README.rst +++ b/elements/package-installs/README.rst @@ -1,8 +1,32 @@ The package-installs element allows for a declarative method of installing and -uninstalling packages for an image build. Adding a file under your elements -pre-install.d, install.d, or post-install.d directories called -package-installs- will cause the list of packages in that file to -be installed at the beginning of the respective phase. +uninstalling packages for an image build. This is done by creating a +package-installs.yaml or package-installs.json file in the element directory. -If the package name in the file starts with a "-", then that package will be -removed at the end of the install.d phase. + +example package-installs.yaml: +libxml2: +grub2: + phase: pre-install.d +networkmanager: + uninstall: True + +example package-installs.json: +{ + "libxml2": null, + "grub2": {"phase": "pre-install.d"}, + "networkmanager": {"uninstall": true} +} + + +Setting phase or uninstall properties for a package overrides the following +default values: + +phase: install.d +uninstall: False + + +DEPRECATED: Adding a file under your elements pre-install.d, install.d, or +post-install.d directories called package-installs- will cause +the list of packages in that file to be installed at the beginning of the +respective phase. If the package name in the file starts with a "-", then +that package will be removed at the end of the install.d phase. diff --git a/elements/package-installs/bin/package-installs-v2 b/elements/package-installs/bin/package-installs-v2 new file mode 100755 index 000000000..86eca98aa --- /dev/null +++ b/elements/package-installs/bin/package-installs-v2 @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +from json import load as json_load +import os +import subprocess +import sys + +from yaml import load as yaml_load + + +class PackageInstalls(object): + + @classmethod + def phase_to_attr(cls, phase): + return phase.replace('.', '_').replace('-', '_') + + @classmethod + def from_yaml_path(cls, path): + with open(path) as fp: + return PackageInstalls.from_native_objs(yaml_load(fp)) + + @classmethod + def from_json_path(cls, path): + with open(path) as fp: + return PackageInstalls.from_native_objs(json_load(fp)) + + @classmethod + def from_native_objs(cls, objs): + init_args = {} + for pkg_name, params in objs.items(): + uninstall = False + phase = "install.d" + + try: + phase = params["phase"] + except (KeyError, TypeError): + pass + + try: + uninstall = bool(params["uninstall"]) + except (KeyError, TypeError): + pass + + init_arg = PackageInstalls.phase_to_attr(phase) + if uninstall: + init_arg = init_arg + '_uninst' + + init_args[init_arg] = init_args.get(init_arg, []) + [pkg_name] + return PackageInstalls(**init_args) + + def __init__(self, **phase_installs): + for phase, pkgs in phase_installs.items(): + setattr(self, phase, pkgs) + + +class PackageInstallsController(object): + + def __init__(self, path='/usr/share/package-installs'): + self.path = path + + def package_installs(self): + for yaml_path in os.listdir(self.path): + full_path = os.path.join(self.path, yaml_path) + if full_path.endswith('.yaml'): + pi = PackageInstalls.from_yaml_path(full_path) + elif full_path.endswith('.json'): + pi = PackageInstalls.from_json_path(full_path) + else: + print("No decoder known for %s, skipping" % full_path) + continue + yield (yaml_path[:-5], pi) + + +def main(): + parser = argparse.ArgumentParser( + description="Install or uninstall packages for a specific phase based" + " on package-installs files.") + parser.add_argument('--phase', + help="Install phase to filter on. Valid options are" + " 'install.d' or pre-install.d") + parser.add_argument('--uninstall', action="store_true", + help="Only show packages to uninstall. By default only" + " packages to install are shown") + args, extra = parser.parse_known_args() + + if not args.phase: + print("Please specify an install phase.") + sys.exit(1) + + pi_c = PackageInstallsController() + pkgs = [] + for element, pi in pi_c.package_installs(): + installs_attr = PackageInstalls.phase_to_attr(args.phase) + if args.uninstall: + installs_attr += '_uninst' + + try: + phase_installs = getattr(pi, installs_attr) + except AttributeError: + continue + + for pkg in phase_installs: + print("Installing %s from %s" % (phase_installs, element)) + pkg_map_args = ["pkg-map", "--missing-ok", "--element", element] + pkg_map_args += phase_installs + + try: + map_output = subprocess.check_output(pkg_map_args) + except subprocess.CalledProcessError as e: + if e.returncode == 1: + print("pkg-map failed with error %s" % e.output) + sys.exit(1) + elif e.returncode == 2: + pkgs += phase_installs + continue + pkgs += map_output.strip().split('\n') + + install_args = ["install-packages"] + if args.uninstall: + install_args.append("-e") + install_args.extend(pkgs) + + try: + subprocess.check_output(install_args) + except subprocess.CalledProcessError as e: + print("install failed with error %s" % e.output) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/elements/package-installs/extra-data.d/11-create-package-installs-dir b/elements/package-installs/extra-data.d/11-create-package-installs-dir new file mode 100755 index 000000000..0aaa50c21 --- /dev/null +++ b/elements/package-installs/extra-data.d/11-create-package-installs-dir @@ -0,0 +1,14 @@ +#!/bin/bash +set -eux +set -o pipefail + +dest_dir="$TMP_MOUNT_PATH/usr/share/package-installs/" +sudo mkdir -p $dest_dir + +for ELEMENT in $IMAGE_ELEMENT ; do + for DIR in ${ELEMENTS_PATH//:/ }; do + if [ -f "$DIR/$ELEMENT/package-installs.yaml" ]; then + sudo cp $DIR/$ELEMENT/package-installs.yaml $dest_dir/$ELEMENT.yaml + fi + done +done diff --git a/elements/package-installs/install.d/00-package-installs b/elements/package-installs/install.d/00-package-installs index 22de5a596..540c8c02c 100755 --- a/elements/package-installs/install.d/00-package-installs +++ b/elements/package-installs/install.d/00-package-installs @@ -4,3 +4,4 @@ set -eux set -o pipefail package-installs -d $(dirname $0) +package-installs-v2 --phase install.d diff --git a/elements/package-installs/install.d/99-package-uninstalls b/elements/package-installs/install.d/99-package-uninstalls index bf8336ee2..b2c91679a 100755 --- a/elements/package-installs/install.d/99-package-uninstalls +++ b/elements/package-installs/install.d/99-package-uninstalls @@ -4,3 +4,4 @@ set -eux set -o pipefail package-uninstalls -d $(dirname $0) +package-installs-v2 --phase install.d --uninstall diff --git a/elements/package-installs/post-install.d/00-package-installs b/elements/package-installs/post-install.d/00-package-installs index 22de5a596..0ed5cdedb 100755 --- a/elements/package-installs/post-install.d/00-package-installs +++ b/elements/package-installs/post-install.d/00-package-installs @@ -4,3 +4,4 @@ set -eux set -o pipefail package-installs -d $(dirname $0) +package-installs-v2 --phase post-install.d diff --git a/elements/package-installs/post-install.d/99-package-uninstalls b/elements/package-installs/post-install.d/99-package-uninstalls index bf8336ee2..f7fb6e02a 100755 --- a/elements/package-installs/post-install.d/99-package-uninstalls +++ b/elements/package-installs/post-install.d/99-package-uninstalls @@ -4,3 +4,4 @@ set -eux set -o pipefail package-uninstalls -d $(dirname $0) +package-installs-v2 --phase post-install.d --uninstall diff --git a/elements/package-installs/pre-install.d/02-package-installs b/elements/package-installs/pre-install.d/02-package-installs index 22de5a596..dbf76c739 100755 --- a/elements/package-installs/pre-install.d/02-package-installs +++ b/elements/package-installs/pre-install.d/02-package-installs @@ -4,3 +4,4 @@ set -eux set -o pipefail package-installs -d $(dirname $0) +package-installs-v2 --phase pre-install.d diff --git a/elements/package-installs/pre-install.d/99-package-uninstalls b/elements/package-installs/pre-install.d/99-package-uninstalls index bf8336ee2..ebccedc01 100755 --- a/elements/package-installs/pre-install.d/99-package-uninstalls +++ b/elements/package-installs/pre-install.d/99-package-uninstalls @@ -4,3 +4,4 @@ set -eux set -o pipefail package-uninstalls -d $(dirname $0) +package-installs-v2 --phase pre-install.d --uninstall diff --git a/tox.ini b/tox.ini index 2bb367b32..0025fc9fa 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,6 @@ commands = bash -c 'if [ ! -d ./.testrepository ] ; then testr init ; fi' downloadcache = ~/cache/pip [flake8] -ignore = E125,H202,H803 +ignore = E125,H202,H302,H803 builtins = _ exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build