mirror/centos: fail on broken/missing package(s)
Apparently yumdownloader does not raise an error if some packages are
missing or broken (dependencies can not be satisfied). As a result some
packages have silently disappered from the mirror (kmod-openvswitch) and
others have been broken (sheepdog).
Patch yumdownloader so it bails out if some packages are missing or have
broken dependencies. As a side effect yumdownloader raises an error if
some of the packages being downloaded conflict (like kernel-headers vs
kernel-headers-lt), so modify the mirroring code to detect and handle
the conflicting packages.
More strict dependencies check yield some unexpected issues. In
particular a few 32 bit packages are broken due to the upgraded base
packages from Fuel repo being x86_64 only. In particular, 32 bit dmraid
package is broken due to device-mapper-libs:
Package: device-mapper-libs-1.02.79-8.el6.i686 (base)
Requires: device-mapper = 1.02.79-8.el6
Available: device-mapper-1.02.79-8.el6.x86_64 (base)
device-mapper = 1.02.79-8.el6
Installing: device-mapper-1.02.90-2.mira1.x86_64 (fuel)
device-mapper = 1.02.90-2.mira1
To solve the problem configure yum to exclude most of i686 packages.
The configuration appeared to be a bit cumbersome since *some* i686
packages are required. In particular syslinux package requires
syslinux-nonlinux (binaries which run in syslinux environment before
booting the OS) which is formally an i686 package.
Closes-Bug: #1457382
Related-Bug: #1456459
Change-Id: I404b662cd3b54ee9d058692bb30177b985871b16
This commit is contained in:
Alexei Sheplyakov
parent
5c8ebddf64
commit
95601a3f4d
|
|
@ -24,6 +24,12 @@ $(BUILD_DIR)/mirror/centos/etc/yum.repos.d/base.repo:
|
|||
@mkdir -p $(@D)
|
||||
/bin/echo -e "$${contents}" > $@
|
||||
|
||||
$(BUILD_DIR)/bin/yumdownloader: $(SOURCE_DIR)/mirror/centos/yumdownloader-deps.patch
|
||||
mkdir -p $(@D)
|
||||
cp -a /usr/bin/yumdownloader $(BUILD_DIR)/yumdownloader
|
||||
( cd $(BUILD_DIR) && patch -p0 ) < $<
|
||||
cp -a $(BUILD_DIR)/yumdownloader $@
|
||||
|
||||
$(BUILD_DIR)/mirror/centos/etc/yum.repos.d/extra.repo: $(call depv,EXTRA_RPM_REPOS)
|
||||
$(BUILD_DIR)/mirror/centos/etc/yum.repos.d/extra.repo: \
|
||||
export contents:=$(foreach repo,$(EXTRA_RPM_REPOS),\n$(call create_extra_repo,repo)\n)
|
||||
|
|
@ -34,6 +40,7 @@ $(BUILD_DIR)/mirror/centos/etc/yum.repos.d/extra.repo:
|
|||
centos_empty_installroot:=$(BUILD_DIR)/mirror/centos/dummy_installroot
|
||||
|
||||
$(BUILD_DIR)/mirror/centos/yum-config.done: \
|
||||
$(BUILD_DIR)/bin/yumdownloader \
|
||||
$(BUILD_DIR)/mirror/centos/etc/yum.conf \
|
||||
$(BUILD_DIR)/mirror/centos/etc/yum.repos.d/base.repo \
|
||||
$(BUILD_DIR)/mirror/centos/etc/yum.repos.d/extra.repo \
|
||||
|
|
@ -62,26 +69,83 @@ $(BUILD_DIR)/mirror/centos/src-rpm-download.done: $(BUILD_DIR)/mirror/centos/src
|
|||
xargs --no-run-if-empty -n1 -P4 wget -Nnv -P "$$dst" < $<
|
||||
$(ACTION.TOUCH)
|
||||
|
||||
|
||||
$(BUILD_DIR)/mirror/centos/urls.list: $(SOURCE_DIR)/requirements-rpm.txt \
|
||||
$(BUILD_DIR)/mirror/centos/yum-config.done
|
||||
# Strip the comments and sort the list alphabetically
|
||||
$(BUILD_DIR)/mirror/centos/requirements-rpm-0.txt: $(SOURCE_DIR)/requirements-rpm.txt
|
||||
mkdir -p $(@D) && \
|
||||
env \
|
||||
TMPDIR="$(centos_empty_installroot)/cache" \
|
||||
TMP="$(centos_empty_installroot)/cache" \
|
||||
yumdownloader -q --urls \
|
||||
--archlist=$(CENTOS_ARCH) \
|
||||
--installroot="$(centos_empty_installroot)" \
|
||||
-c $(BUILD_DIR)/mirror/centos/etc/yum.conf \
|
||||
--resolve \
|
||||
`cat $(SOURCE_DIR)/requirements-rpm.txt` > "$@.out" 2>"$@.log"
|
||||
# yumdownloader -q prints logs to stdout, filter them out
|
||||
grep -v -e '^#' $< > $@.tmp && \
|
||||
sort -u < $@.tmp > $@.pre && \
|
||||
mv $@.pre $@
|
||||
|
||||
$(BUILD_DIR)/mirror/centos/urls.list: $(BUILD_DIR)/mirror/centos/requirements-rpm-0.txt \
|
||||
$(BUILD_DIR)/mirror/centos/yum-config.done
|
||||
touch "$(BUILD_DIR)/mirror/centos/conflicting-packages-0.lst"
|
||||
# 1st pass - find out which packages conflict
|
||||
# 2nd pass - get the URLs of non-conflicting packages
|
||||
# 3rd pass (under the else clause) - process the conflicting rpms one by one
|
||||
count=0; \
|
||||
while true; do \
|
||||
if [ $$count -gt 1 ]; then \
|
||||
echo "Unable to resolve packages dependencies" >&2; \
|
||||
cat $(BUILD_DIR)/mirror/centos/yumdownloader-1.out >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
requirements_rpm="$(BUILD_DIR)/mirror/centos/requirements-rpm-$${count}.txt"; \
|
||||
requirements_rpm_next="$(BUILD_DIR)/mirror/centos/requirements-rpm-$$((count+1)).txt"; \
|
||||
out="$(BUILD_DIR)/mirror/centos/yumdownloader-$${count}.out"; \
|
||||
log="$(BUILD_DIR)/mirror/centos/yumdownloader-$${count}.log"; \
|
||||
conflict_lst="$(BUILD_DIR)/mirror/centos/conflicting-packages-$${count}.lst"; \
|
||||
conflict_lst_next="$(BUILD_DIR)/mirror/centos/conflicting-packages-$$((count+1)).lst"; \
|
||||
if ! env \
|
||||
TMPDIR="$(centos_empty_installroot)/cache" \
|
||||
TMP="$(centos_empty_installroot)/cache" \
|
||||
$(BUILD_DIR)/bin/yumdownloader -q --urls \
|
||||
--archlist=$(CENTOS_ARCH) \
|
||||
--installroot="$(centos_empty_installroot)" \
|
||||
-c $(BUILD_DIR)/mirror/centos/etc/yum.conf \
|
||||
--resolve \
|
||||
`cat $${requirements_rpm}` > "$$out" 2>"$$log"; then \
|
||||
sed -rne 's/^([a-zA-Z0-9_-]+)\s+conflicts with\s+(.+)$$/\1/p' < "$$out" > "$${conflict_lst_next}.pre" && \
|
||||
# Package X can declare conflict with package Y; but package Y is not obliged \
|
||||
# to declare a conflict with package X. yum will report that X conflicts with Y. \
|
||||
# We need to figure out that Y conflicts with X on our own. \
|
||||
sed -rne 's/^([a-zA-Z0-9_-]+)\s+conflicts with\s+(.+)$$/\2/p' < "$$out" > "$${conflict_lst_next}.more" && \
|
||||
while read nvra; do \
|
||||
nvr="$${nvra%.*}"; nv="$${nvr%-*}"; n="$${nv%-*}"; echo $$n; \
|
||||
done < "$${conflict_lst_next}.more" >> "$${conflict_lst_next}.pre" && \
|
||||
cat "$${conflict_lst_next}.pre" "$$conflict_lst" | sort -u > "$$conflict_lst_next" && \
|
||||
comm -23 "$$requirements_rpm" "$$conflict_lst_next" > "$${requirements_rpm}.new.pre" && \
|
||||
sort -u < "$${requirements_rpm}.new.pre" > "$${requirements_rpm_next}"; \
|
||||
else \
|
||||
conflicting_pkgs_urls="$(BUILD_DIR)/mirror/centos/urls_conflicting.lst"; \
|
||||
nonconflicting_pkgs="$$requirements_rpm"; \
|
||||
# Now process conflicting packages one by one. There is a small problem: \
|
||||
# in the original requirements-rpm.txt quite a number of packages are \
|
||||
# pinned to specific versions. These pins should be taken into account \
|
||||
# to avoid having several versions of the same package. For instance, \
|
||||
# zabbix-web-* depends on httpd, so the latest version of httpd gets \
|
||||
# installed along with the one listed in the requirements-rpm.txt. \
|
||||
# Therefore add the set of all nonconflicting packages to the package \
|
||||
# being processed to take into account version pins. \
|
||||
for pkg in `cat $$conflict_lst`; do \
|
||||
if ! env \
|
||||
TMPDIR="$(centos_empty_installroot)/cache" \
|
||||
TMP="$(centos_empty_installroot)/cache" \
|
||||
$(BUILD_DIR)/bin/yumdownloader -q --urls \
|
||||
--archlist=$(CENTOS_ARCH) \
|
||||
--installroot="$(centos_empty_installroot)" \
|
||||
-c "$(BUILD_DIR)/mirror/centos/etc/yum.conf" \
|
||||
--resolve $$pkg `cat $$nonconflicting_pkgs`; then \
|
||||
echo "Failed to resolve package $$pkg" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
done > "$$conflicting_pkgs_urls" && \
|
||||
cat "$$out" "$$conflicting_pkgs_urls" > "$@.out" && \
|
||||
break; \
|
||||
fi; \
|
||||
count=$$((count+1)); \
|
||||
done
|
||||
# yumdownloader -q prints logs to stdout, filter them out
|
||||
sed -rne '/\.rpm$$/ {p}' < $@.out > $@.pre
|
||||
# yumdownloader selects i686 packages too. Remove them. However be
|
||||
# careful not to remove the syslinux-nolinux package (it contains
|
||||
# 32 binaries executed on a bare hardware. That package really should
|
||||
# have been noarch
|
||||
sed -re '/i686\.rpm$$/ { /syslinux-nonlinux/p;d }' -i $@.pre
|
||||
sort -u < $@.pre > $@.tmp
|
||||
mv $@.tmp $@
|
||||
|
||||
|
|
@ -95,7 +159,7 @@ $(BUILD_DIR)/mirror/centos/src_urls.list: $(BUILD_DIR)/mirror/centos/mirantis_rp
|
|||
env \
|
||||
TMPDIR="$(centos_empty_installroot)/cache" \
|
||||
TMP="$(centos_empty_installroot)/cache" \
|
||||
yumdownloader -q --urls \
|
||||
$(BUILD_DIR)/bin/yumdownloader -q --urls \
|
||||
--archlist=src --source \
|
||||
--installroot="$(centos_empty_installroot)" \
|
||||
-c $(BUILD_DIR)/mirror/centos/etc/yum.conf \
|
||||
|
|
|
|||
|
|
@ -1,10 +1,32 @@
|
|||
|
||||
# Problem: --archlist=x86_64 really means "x86_64 and i686". Therefore yum
|
||||
# tries to resolve dependencies of i686 packages. Sometimes this fails due
|
||||
# to an upgraded x86_64 only package available in the fuel repo. For
|
||||
# instance, when yum is asked to download dmraid package it tries to resolve
|
||||
# the dependencies of i686 version. This fails since the upgraded
|
||||
# device-mapper-libs package (from the fuel repo) is x86_64 only:
|
||||
# Package: device-mapper-libs-1.02.79-8.el6.i686 (base)
|
||||
# Requires: device-mapper = 1.02.79-8.el6
|
||||
# Available: device-mapper-1.02.79-8.el6.x86_64 (base)
|
||||
# device-mapper = 1.02.79-8.el6
|
||||
# Installing: device-mapper-1.02.90-2.mira1.x86_64 (fuel)
|
||||
# device-mapper = 1.02.90-2.mira1
|
||||
# The obvious solution is to exclude i686 packages. However syslinux
|
||||
# package depends on i686 package syslinux-nonlinux (which contians
|
||||
# the binaries that run in the syslinux environment). Since excluding
|
||||
# packages by regexp is impossible (only glob patterns are supported)
|
||||
# base and updates repos are "cloned". Those "cloned" repos contain
|
||||
# a few whitelisted i686 packages (for now only syslinux).
|
||||
# Note: these packages should be also excluded from base and updates.
|
||||
x86_rpm_packages_whitelist:=syslinux*
|
||||
|
||||
define yum_conf
|
||||
[main]
|
||||
cachedir=$(BUILD_DIR)/mirror/centos/cache
|
||||
keepcache=0
|
||||
debuglevel=6
|
||||
logfile=$(BUILD_DIR)/mirror/centos/yum.log
|
||||
exclude=*.i686.rpm ntp-dev*
|
||||
exclude=ntp-dev*
|
||||
exactarch=1
|
||||
obsoletes=1
|
||||
gpgcheck=0
|
||||
|
|
@ -21,6 +43,7 @@ name=CentOS-$(CENTOS_RELEASE) - Base
|
|||
baseurl=$(MIRROR_CENTOS)/os/$(CENTOS_ARCH)
|
||||
gpgcheck=0
|
||||
enabled=1
|
||||
exclude=*i686 $(x86_rpm_packages_whitelist)
|
||||
priority=10
|
||||
|
||||
[updates]
|
||||
|
|
@ -29,6 +52,25 @@ name=CentOS-$(CENTOS_RELEASE) - Updates
|
|||
baseurl=$(MIRROR_CENTOS)/updates/$(CENTOS_ARCH)
|
||||
gpgcheck=0
|
||||
enabled=1
|
||||
exclude=*i686 $(x86_rpm_packages_whitelist)
|
||||
priority=10
|
||||
|
||||
[base_i686_whitelisted]
|
||||
name=CentOS-$(CENTOS_RELEASE) - Base
|
||||
#mirrorlist=http://mirrorlist.centos.org/?release=$(CENTOS_RELEASE)&arch=$(CENTOS_ARCH)&repo=os
|
||||
baseurl=$(MIRROR_CENTOS)/os/$(CENTOS_ARCH)
|
||||
gpgcheck=0
|
||||
enabled=1
|
||||
includepkgs=$(x86_rpm_packages_whitelist)
|
||||
priority=10
|
||||
|
||||
[updates_i686_whitelisted]
|
||||
name=CentOS-$(CENTOS_RELEASE) - Updates
|
||||
#mirrorlist=http://mirrorlist.centos.org/?release=$(CENTOS_RELEASE)&arch=$(CENTOS_ARCH)&repo=updates
|
||||
baseurl=$(MIRROR_CENTOS)/updates/$(CENTOS_ARCH)
|
||||
gpgcheck=0
|
||||
enabled=1
|
||||
includepkgs=$(x86_rpm_packages_whitelist)
|
||||
priority=10
|
||||
|
||||
[extras]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
--- yumdownloader.orig 2013-11-15 00:01:02.000000000 +0400
|
||||
+++ yumdownloader 2015-05-21 12:48:25.877327687 +0300
|
||||
@@ -182,7 +182,7 @@
|
||||
|
||||
if not installable: # doing one at a time, apart from groups
|
||||
self.logger.error('No Match for argument %s' % pkg)
|
||||
- continue
|
||||
+ sys.exit(1)
|
||||
for newpkg in installable:
|
||||
toActOn.extend(_best_convert_pkg2srcpkgs(self, opts, newpkg))
|
||||
if toActOn:
|
||||
@@ -208,7 +208,12 @@
|
||||
self.tsInfo.addInstall(po)
|
||||
self.localPackages.append(po)
|
||||
# Resolve dependencies
|
||||
- self.resolveDeps()
|
||||
+ result, errmsgs = self.resolveDeps()
|
||||
+ if result == 1:
|
||||
+ self.logger.error('Dependency resolution failed')
|
||||
+ for msg in errmsgs:
|
||||
+ self.logger.error(msg)
|
||||
+ sys.exit(1)
|
||||
# Add newly added packages to the toDownload list
|
||||
for pkg in self.tsInfo.getMembers():
|
||||
if not pkg in toDownload:
|
||||
|
|
@ -437,7 +437,7 @@ scapy-2.0.0.10-5.el6
|
|||
screen-4.0.3-16.el6
|
||||
selinux-policy
|
||||
send2syslog-0.1-1.el6
|
||||
sheepdog
|
||||
#sheepdog
|
||||
shotgun
|
||||
snappy-1.0.5-1.el6
|
||||
socat-1.7.2.2-1.el6
|
||||
|
|
|
|||
Loading…
Reference in New Issue