x/anvil
Code Issues Proposed changes

Move package management to yum tool

Browse Source 
This commit introduces new tool named YYOOM that handles package
management. It uses yum python module (and so is GPLv2+ licensed)
and print work result in JSON to stdout.

Anvil uses it to install packages and query which packages are
install or available. We also log all installed packages (including
deps) via tracewriter, which allows to use tracereader for
clean and complete uninstall.

Fixes: bug 1189707
Change-Id: Ib6d13b2dc816a3d2f8875aa23779e34fa685cd31
This commit is contained in:
Ivan A. Melnikov 2013-07-08 20:11:42 +04:00
parent 5796566830
commit 07eb723261
6 changed files with 392 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,6 +3,7 @@
*.pyc
/build/
/dist/
/tools/yyoomc
.coverage
*.egg-info

10
README.rst
View File

@ -2,3 +2,13 @@ We want more information!
=========================
Please check out: http://anvil.readthedocs.org.
Licensing
=========
Anvil is licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You may obtain a
copy of the License at http://www.apache.org/licenses/LICENSE-2.0
Some tools are licensed under different terms; see tools/README.rst for
more information.

107
anvil/packaging/helpers/yum_helper.py
View File

@ -14,47 +14,40 @@
# License for the specific language governing permissions and limitations
# under the License.
# See http://yum.baseurl.org/api/yum-3.2.26/yum-module.html
from yum import YumBase
import sys
import json
from yum.packages import PackageObject
from anvil import log as logging
from anvil import shell as sh
class Requirement(object):
def __init__(self, name, version):
self.name = str(name)
self.version = version
def __str__(self):
name = self.name
if self.version is not None:
name += "-%s" % (self.version)
return name
@property
def package(self):
# Form a 'fake' rpm package that
# can be used to compare against
# other rpm packages using the
# standard rpm routines
my_pkg = PackageObject()
my_pkg.name = self.name
if self.version is not None:
my_pkg.version = str(self.version)
return my_pkg
LOG = logging.getLogger(__name__)
class Helper(object):
# Cache of yumbase object
_yum_base = None
def __init__(self):
self._installed = None
self._available = None
@staticmethod
def _get_yum_base():
if Helper._yum_base is None:
_yum_base = YumBase()
_yum_base.setCacheDir(force=True)
Helper._yum_base = _yum_base
return Helper._yum_base
def _yyoom(arglist):
executable = sh.which("yyoom", ["tools/"])
cmdline = [executable]
if LOG.logger.isEnabledFor(logging.DEBUG):
cmdline.append('--verbose')
cmdline.extend(arglist)
out = sh.execute(cmdline, stderr_fh=sys.stderr)[0].strip()
if out:
return json.loads(out)
return None
@staticmethod
def _trace_installed_packages(tracewriter, data):
if tracewriter is None or not data:
return
for action in data:
if action['action_type'] == 'install':
tracewriter.package_installed(action['name'])
def is_installed(self, name):
if len(self.get_installed(name)):
@ -63,18 +56,38 @@ class Helper(object):
return False
def get_available(self):
base = Helper._get_yum_base()
pkgs = base.doPackageLists(showdups=True)
avail = list(pkgs.available)
avail.extend(pkgs.installed)
return avail
if self._available is None:
self._available = self._yyoom(['list', 'available'])
return self._available
def get_installed(self, name):
base = Helper._get_yum_base()
pkgs = base.doPackageLists(pkgnarrow='installed',
ignore_case=True, patterns=[name])
if pkgs.installed:
whats_installed = list(pkgs.installed)
else:
whats_installed = []
return whats_installed
if self._installed is None:
self._installed = self._yyoom(['list', 'installed'])
return [item for item in self._installed
if item['name'] == name]
def builddep(self, srpm_path, tracewriter=None):
data = self._yyoom(['builddep', srpm_path])
self._trace_installed_packages(tracewriter, data)
def clean(self):
self._yyoom(['cleanall'])
def transaction(self, install_pkgs=(), remove_pkgs=(), tracewriter=None):
if not install_pkgs and not remove_pkgs:
return
# reset the caches:
self._installed = None
self._available = None
cmdline = ['transaction']
for pkg in install_pkgs:
cmdline.append('--install')
cmdline.append(pkg)
for pkg in remove_pkgs:
cmdline.append('--erase')
cmdline.append(pkg)
data = self._yyoom(cmdline)
self._trace_installed_packages(tracewriter, data)

74
anvil/packaging/yum.py
View File

@ -16,7 +16,6 @@
import collections
import contextlib
import os
import pkg_resources
import sys
@ -25,7 +24,6 @@ import rpm
import tarfile
from anvil import colorizer
from anvil import env
from anvil import exceptions as excp
from anvil import log as logging
from anvil.packaging import base
@ -103,18 +101,6 @@ class YumDependencyHandler(base.DependencyHandler):
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
self._no_remove = None
@property
def no_remove(self):
if self._no_remove is not None:
return self._no_remove
packages = env.get_key('REQUIRED_PACKAGES', default_value='').split()
own_details = pip_helper.get_directory_details(os.getcwd())
required_pips = own_details['dependencies']
no_remove = self._convert_names_python2rpm(required_pips)
no_remove.extend(packages)
self._no_remove = no_remove
return self._no_remove
def py2rpm_start_cmdline(self):
cmdline = [
self.py2rpm_executable,
@ -178,20 +164,13 @@ class YumDependencyHandler(base.DependencyHandler):
sh.move(filename, target_dir, force=True)
def build_binary(self):
def _install_build_requirements():
build_requires = self.requirements["build-requires"]
if build_requires:
utils.log_iterable(sorted(build_requires),
header=("Installing %s build requirements" % len(build_requires)),
logger=LOG)
cmdline = ["yum", "install", "-y"] + list(build_requires)
sh.execute(cmdline)
def _is_src_rpm(filename):
return filename.endswith('.src.rpm')
_install_build_requirements()
LOG.info("Installing build requirements")
self.helper.transaction(
install_pkgs=self.requirements["build-requires"],
tracewriter=self.tracewriter)
for repo_name in self.REPOS:
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
@ -269,8 +248,8 @@ class YumDependencyHandler(base.DependencyHandler):
def _get_yum_available(self):
yum_map = collections.defaultdict(list)
for pkg in self.helper.get_available():
for provides in pkg.provides:
yum_map[provides[0]].append((pkg.version, pkg.repo))
for provides in pkg['provides']:
yum_map[provides[0]].append((pkg['version'], pkg['repo']))
return dict(yum_map)
@staticmethod
@ -638,39 +617,18 @@ class YumDependencyHandler(base.DependencyHandler):
def install(self):
super(YumDependencyHandler, self).install()
self.helper.clean()
# Erase conflicting packages
cmdline = []
for p in self.requirements["conflicts"]:
if self.helper.is_installed(p):
cmdline.append(p)
if cmdline:
cmdline = ["yum", "erase", "-y"] + cmdline
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
cmdline = ["yum", "clean", "all"]
sh.execute(cmdline)
rpm_names = self._all_rpm_names()
if rpm_names:
cmdline = ["yum", "install", "-y"] + rpm_names
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
remove_pkgs = [pkg_name
for pkg_name in self.requirements["conflicts"]
if self.helper.is_installed(pkg_name)]
self.helper.transaction(install_pkgs=self._all_rpm_names(),
remove_pkgs=remove_pkgs,
tracewriter=self.tracewriter)
def uninstall(self):
super(YumDependencyHandler, self).uninstall()
scan_packages = self._all_rpm_names()
rpm_names = []
for p in scan_packages:
if p in self.no_remove:
continue
if self.helper.is_installed(p):
rpm_names.append(p)
if rpm_names:
cmdline = ["yum", "remove", "--remove-leaves", "-y"]
for p in self.no_remove:
cmdline.append("--exclude=%s" % (p))
cmdline.extend(sorted(set(rpm_names)))
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
if self.tracereader.exists():
remove_pkgs = self.tracereader.packages_installed()
self.helper.transaction(remove_pkgs=remove_pkgs)

14
tools/README.rst
View File

@ -95,6 +95,20 @@ builds RPMs (current directory is used by default)::
...
yyoom
-----
`yyoom` uses yum API to provide nice command-line interface to package
management. It is able to install and remove packages in the same
transaction (see `yyoom transaction --help`), list available or installed
packages and a bit more. It writes results of its work to standard output
in JSON.
`yyoom` is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
git-changelog
-------------
This tool generates a pretty software's changelog from git history.

291
tools/yyoom Executable file
View File

@ -0,0 +1,291 @@
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2005 Duke University
# Parts Copyright 2007 Red Hat, Inc
"""YYOOM: a package management utility
Using Yum API instead of /usr/bin/yum provides several interesting
capabilities, some of which we are desperate to use, including:
- installing and removing packages in same transaction;
- JSON output.
"""
import argparse
import json
import logging
import os
import sys
import yum
from contextlib import contextmanager
LOG = logging.getLogger('yum-tool')
OUTPUT = None
def _setup_output():
"""Do some nasty manipulations with fds
Yum internals may sometimes write to stdout, just out of a sudden.
To prevent this output form interfering with our JSON, we save
current stdout to other fd via os.dup, and replace fd 1 with
/dev/null opened for writing.
"""
global OUTPUT
# save current stdout for later use
OUTPUT = os.fdopen(os.dup(sys.stdout.fileno()), 'wb')
# close the stream
sys.stdout.close()
# open /dev/null -- all writes to stdout from now on will go there
devnull_fd = os.open(os.devnull, os.O_WRONLY)
if devnull_fd != 1:
os.dup2(devnull_fd, 1)
os.close(devnull_fd)
sys.stdout = os.fdopen(1, 'w')
def _write_output(data):
"""Dump given object as pretty json"""
OUTPUT.write(json.dumps(data, indent=4,
separators=(',', ': '),
sort_keys=True) + '\n')
def _package_info(pkg, **kwargs):
if isinstance(pkg, basestring):
result = dict(name=pkg, **kwargs)
else:
result = dict(
name=pkg.name,
epoch=pkg.epoch,
version=pkg.version,
release=pkg.release,
provides=pkg.provides,
repo=str(pkg.repo),
arch=pkg.arch,
**kwargs
)
return result
class _RPMCallback(yum.rpmtrans.RPMBaseCallback):
"""Listen to events from RPM transactions"""
def __init__(self):
self.seen = []
def event(self, package, action, te_current, te_total,
ts_current, ts_total):
pass
def scriptout(self, package, msg):
if not msg or not LOG.isEnabledFor(logging.INFO):
return
for line in msg.splitlines():
line = line.strip()
if line:
LOG.info("%s: %s", package, line)
def errorlog(self, msg):
LOG.error("%s", msg)
def filelog(self, package, action):
action_data = _package_info(package, action_code=action)
if action in yum.constants.TS_INSTALL_STATES:
action_data['action_type'] = 'install'
elif action in yum.constants.TS_REMOVE_STATES:
action_data['action_type'] = 'erase'
else:
action_data['action_type'] = 'other'
self.seen.append(action_data)
LOG.info("Performed %(action_type)s (code %(action_code)s) on %(name)s"
% action_data)
def _run(yum_base, options):
"""Handler of `transaction` command
Installs and erases packages, prints what was done in JSON
"""
LOG.debug('Erasing packages: %s', options.erase)
LOG.debug('Installing packages: %s', options.install)
callback = _RPMCallback()
with _transaction(yum_base, callback):
for name in options.erase or ():
yum_base.remove(name=name)
for name in options.install or ():
yum_base.install(name=name)
_write_output(callback.seen)
def _list(yum_base, options):
"""Handler of `list` command"""
pkgnarrow = options.what[0] if len(options.what) == 1 else 'all'
lists = yum_base.doPackageLists(pkgnarrow=pkgnarrow, showdups=True)
LOG.debug("Got packages for '%s': %s installed, %s available,"
"%s available for reinstall, %s extras",
pkgnarrow, len(lists.installed), len(lists.available),
len(lists.reinstall_available), len(lists.extras))
result = []
if 'installed' in options.what:
result.extend(_package_info(pkg, status='installed')
for pkg in lists.installed)
if 'available' in options.what:
result.extend(_package_info(pkg, status='available')
for pkg in lists.available)
result.extend(_package_info(pkg, status='available')
for pkg in lists.reinstall_available)
if 'extras' in options.what:
result.extend(_package_info(pkg, status='installed')
for pkg in lists.extras)
_write_output(result)
def _cleanall(yum_base, options):
"""Handler of `cleanall` command"""
LOG.info("Running yum cleanup")
code = sum((
_run_yum_api('packages clean up', yum_base.cleanPackages),
_run_yum_api('headers clean up', yum_base.cleanHeaders),
_run_yum_api('metadata clean up', yum_base.cleanMetadata),
_run_yum_api('sqlite clean up', yum_base.cleanSqlite),
_run_yum_api('rpm db clean up', yum_base.cleanRpmDB),
))
return code
def _builddep(yum_base, options):
"""Handler of `builddep` command
Installs build dependencies for given package, prints what was done
in JSON.
"""
LOG.info("Installing build dependencies for package %s", options.srpm)
srpm = yum.packages.YumLocalPackage(yum_base.ts, options.srpm)
callback = _RPMCallback()
with _transaction(yum_base, callback):
for req in srpm.requiresList():
LOG.debug('Processing dependency: %s', req)
if not (
req.startswith('rpmlib(') or
yum_base.returnInstalledPackagesByDep(req)
):
pkg = yum_base.returnPackageByDep(req)
LOG.debug('Installing %s', pkg)
yum_base.install(pkg)
_write_output(callback.seen)
def _parse_arguments(args):
parser = argparse.ArgumentParser(prog=args[0])
parser.add_argument('--verbose', '-v', action='store_true',
help='verbose operation')
# TODO(imelnikov): --format
subparsers = parser.add_subparsers(title='subcommands')
parser_list = subparsers.add_parser('list', help='list packages')
parser_list.add_argument('what', nargs='+',
choices=('installed', 'available', 'extras'),
help='what packages to list')
parser_list.set_defaults(func=_list)
parser_run = subparsers.add_parser('transaction',
help='install or remove packages')
parser_run.set_defaults(func=_run)
parser_run.add_argument('--install', '-i', action='append',
metavar='package',
help='install package')
parser_run.add_argument('--erase', '-e', action='append',
metavar='package',
help='erase package')
parser_builddep = subparsers.add_parser(
'builddep', help='install build dependencies of srpm')
parser_builddep.add_argument('srpm', help='path to source RPM package')
parser_builddep.set_defaults(func=_builddep)
parser_cleanall = subparsers.add_parser('cleanall', help='clean all')
parser_cleanall.set_defaults(func=_cleanall)
return parser.parse_args(args[1:])
def _setup_logging(verbose=True):
"""Initialize logging"""
# setup logging -- put messages to stderr
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(logging.Formatter('YYOOM %(levelname)s: %(message)s'))
root_logger = logging.getLogger()
root_logger.addHandler(handler)
root_logger.setLevel(logging.DEBUG if verbose else logging.INFO)
def _get_yum_base():
base = yum.YumBase()
base.setCacheDir(force=True)
return base
def _run_yum_api(name, func, ok_codes=(0,), *args, **kwargs):
code, results = func(*args, **kwargs)
for msg in results:
LOG.debug(msg)
if code not in ok_codes:
LOG.error('%s failed', name.title())
return code
@contextmanager
def _transaction(base, callback):
"""Manage Yum transactions
Locks and unlocks Yum database, builds and processes transaction
on __exit__.
"""
try:
base.doLock()
yield
code = _run_yum_api('building transaction',
base.buildTransaction, ok_codes=(0, 2))
if code == 0:
LOG.debug('Nothing to do')
elif code == 2:
base.processTransaction(rpmTestDisplay=callback,
rpmDisplay=callback)
else:
raise RuntimeError("Transaction failed: %s" % code)
finally:
del base.tsInfo
del base.ts
base.doUnlock()
def main(args):
options = _parse_arguments(args)
try:
_setup_output()
_setup_logging(options.verbose)
return options.func(_get_yum_base(), options) or 0
except Exception as e:
if options.verbose:
raise # let python runtime write stacktrace
sys.stderr.write("Failed: %s\n" % e)
return 1
if __name__ == '__main__':
sys.exit(main(sys.argv))