openstack/fuel-main
Code Issues Proposed changes

packages/deb: speed up the build

Browse Source 
Create a tarball of a base system + build-essential packages, and use that
for building the packages instead of bootstrapping Ubuntu from the scratch
for every package.

Change-Id: I91a1aa326fc8de51e1e5305fbe8b9dd3ae6c226e
This commit is contained in:
Alexei Sheplyakov 2015-01-06 21:13:01 +03:00
parent 85c914d12f
commit 1784280ee4
2 changed files with 31 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
packages/deb/module.mk
View File

@ -3,13 +3,24 @@
clean: clean-deb clean: clean-deb
clean-deb: clean-deb:
-mount | grep '$(BUILD_DIR)/packages/deb/SANDBOX' | while read entry; do \ -mount | grep '$(BUILD_DIR)/packages/deb' | while read entry; do \
set -- $$entry; \ set -- $$entry; \
mntpt="$$3"; \ mntpt="$$3"; \
sudo umount $$mntpt; \ sudo umount $$mntpt; \
done done
sudo rm -rf $(BUILD_DIR)/packages/deb sudo rm -rf $(BUILD_DIR)/packages/deb
$(BUILD_DIR)/packages/deb/buildd.tar.gz: SANDBOX_DEB_PKGS:=wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot
$(BUILD_DIR)/packages/deb/buildd.tar.gz: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/chroot
$(BUILD_DIR)/packages/deb/buildd.tar.gz: export SANDBOX_UBUNTU_UP:=$(SANDBOX_UBUNTU_UP)
$(BUILD_DIR)/packages/deb/buildd.tar.gz: export SANDBOX_UBUNTU_DOWN:=$(SANDBOX_UBUNTU_DOWN)
$(BUILD_DIR)/packages/deb/buildd.tar.gz: $(BUILD_DIR)/mirror/ubuntu/build.done
sh -c "$${SANDBOX_UBUNTU_UP}"
sh -c "$${SANDBOX_UBUNTU_DOWN}"
sudo rm -f $(SANDBOX_UBUNTU)/var/cache/apt/archives/*.deb
sudo tar czf $@.tmp -C $(SANDBOX_UBUNTU) .
mv $@.tmp $@
# Usage: # Usage:
# (eval (call build_deb,package_name)) # (eval (call build_deb,package_name))
define build_deb define build_deb
@ -17,15 +28,21 @@ $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1.done
$(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1-repocleanup.done $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1-repocleanup.done
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/mirror/ubuntu/build.done $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/mirror/ubuntu/build.done
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/packages/source_$1.done $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/packages/source_$1.done
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/packages/deb/buildd.tar.gz
$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX/$1 $(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX/$1
$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_DEB_PKGS:=apt wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot $(BUILD_DIR)/packages/deb/$1.done: SANDBOX_DEB_PKGS:=apt wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot
$(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_UP:=$$(SANDBOX_UBUNTU_UP) $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_UP:=$$(SANDBOX_UBUNTU_UP)
$(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_DOWN:=$$(SANDBOX_UBUNTU_DOWN) $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_DOWN:=$$(SANDBOX_UBUNTU_DOWN)
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/repos/repos.done $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/repos/repos.done
mkdir -p $(BUILD_DIR)/packages/deb/packages $(BUILD_DIR)/packages/deb/sources mkdir -p $(BUILD_DIR)/packages/deb/packages $(BUILD_DIR)/packages/deb/sources
# create sandbox if does not exist mkdir -p "$$(SANDBOX_UBUNTU)" && mkdir -p "$$(SANDBOX_UBUNTU)/tmp/apt"
test -e $$(SANDBOX_UBUNTU)/etc/debian_version || sudo sh -c "$$$${SANDBOX_UBUNTU_UP}" if [ ! -e "$$(SANDBOX_UBUNTU)/etc/debian_version" ]; then \
# mount proc if not mounted sudo tar xaf $(BUILD_DIR)/packages/deb/buildd.tar.gz -C $$(SANDBOX_UBUNTU); \
fi
if ! mountpoint -q $$(SANDBOX_UBUNTU)/tmp/apt; then \
sudo mount -o bind $(LOCAL_MIRROR_UBUNTU) $$(SANDBOX_UBUNTU)/tmp/apt; \
sudo mount -o remount,ro,bind $$(SANDBOX_UBUNTU)/tmp/apt; \
fi
mountpoint -q $$(SANDBOX_UBUNTU)/proc || sudo mount -t proc sandbox_ubuntu_proc $$(SANDBOX_UBUNTU)/proc mountpoint -q $$(SANDBOX_UBUNTU)/proc || sudo mount -t proc sandbox_ubuntu_proc $$(SANDBOX_UBUNTU)/proc
sudo mkdir -p $$(SANDBOX_UBUNTU)/tmp/$1 sudo mkdir -p $$(SANDBOX_UBUNTU)/tmp/$1
ifeq ($1,$(filter $1,nailgun-net-check python-tasklib)) ifeq ($1,$(filter $1,nailgun-net-check python-tasklib))
@ -35,10 +52,9 @@ endif
sudo cp -r $(BUILD_DIR)/packages/sources/$1/* $$(SANDBOX_UBUNTU)/tmp/$1/ sudo cp -r $(BUILD_DIR)/packages/sources/$1/* $$(SANDBOX_UBUNTU)/tmp/$1/
sudo cp -r $(SOURCE_DIR)/packages/deb/specs/$1/* $$(SANDBOX_UBUNTU)/tmp/$1/ sudo cp -r $(SOURCE_DIR)/packages/deb/specs/$1/* $$(SANDBOX_UBUNTU)/tmp/$1/
dpkg-checkbuilddeps $(SOURCE_DIR)/packages/deb/specs/$1/debian/control 2>&1 | sed 's/^dpkg-checkbuilddeps: Unmet build dependencies: //g' | sed 's/([^()]*)//g;s/|//g' | sudo tee $$(SANDBOX_UBUNTU)/tmp/$1.installdeps dpkg-checkbuilddeps $(SOURCE_DIR)/packages/deb/specs/$1/debian/control 2>&1 | sed 's/^dpkg-checkbuilddeps: Unmet build dependencies: //g' | sed 's/([^()]*)//g;s/|//g' | sudo tee $$(SANDBOX_UBUNTU)/tmp/$1.installdeps
sudo chroot $$(SANDBOX_UBUNTU) /bin/bash -c "cat /tmp/$1.installdeps | xargs apt-get -y install" sudo chroot $$(SANDBOX_UBUNTU) /bin/sh -c "cat /tmp/$1.installdeps | xargs apt-get -y install"
sudo chroot $$(SANDBOX_UBUNTU) /bin/bash -c "cd /tmp/$1 ; DEB_BUILD_OPTIONS=nocheck debuild -us -uc -b -d" sudo chroot $$(SANDBOX_UBUNTU) /bin/sh -c "cd /tmp/$1 ; DEB_BUILD_OPTIONS=nocheck debuild -us -uc -b -d"
cp $$(SANDBOX_UBUNTU)/tmp/*$1*.deb $(BUILD_DIR)/packages/deb/packages cp $$(SANDBOX_UBUNTU)/tmp/*$1*.deb $(BUILD_DIR)/packages/deb/packages
sudo rm -rf $$(SANDBOX_UBUNTU)/tmp/*
sudo sh -c "$$$${SANDBOX_UBUNTU_DOWN}" sudo sh -c "$$$${SANDBOX_UBUNTU_DOWN}"
$$(ACTION.TOUCH) $$(ACTION.TOUCH)
@ -47,11 +63,9 @@ $(BUILD_DIR)/packages/deb/$1-repocleanup.done: $(BUILD_DIR)/mirror/ubuntu/build.
$$(ACTION.TOUCH) $$(ACTION.TOUCH)
endef endef
$(eval $(call build_deb,fencing-agent))
$(eval $(call build_deb,nailgun-mcagents)) fuel_debian_packages:=fencing-agent nailgun-mcagents nailgun-net-check nailgun-agent python-tasklib
$(eval $(call build_deb,nailgun-net-check)) $(eval $(foreach pkg,$(fuel_debian_packages),$(call build_deb,$(pkg))$(NEWLINE)))
$(eval $(call build_deb,nailgun-agent))
$(eval $(call build_deb,python-tasklib))
$(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/repo.done:
sudo find $(BUILD_DIR)/packages/deb/packages -name '*.deb' -exec cp -u {} $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/pool/main \; sudo find $(BUILD_DIR)/packages/deb/packages -name '*.deb' -exec cp -u {} $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/pool/main \;

10
sandbox.mk
View File

@ -69,10 +69,10 @@ sudo cp /etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf
echo "Generating utf8 locale" echo "Generating utf8 locale"
sudo chroot $(SANDBOX_UBUNTU) /bin/sh -c 'locale-gen en_US.UTF-8; dpkg-reconfigure locales' sudo chroot $(SANDBOX_UBUNTU) /bin/sh -c 'locale-gen en_US.UTF-8; dpkg-reconfigure locales'
echo "Preparing directory for chroot local mirror" echo "Preparing directory for chroot local mirror"
test -e $(SANDBOX_UBUNTU)/tmp/apt && sudo rm -rf $(SANDBOX_UBUNTU)/tmp/apt
sudo mkdir -p $(SANDBOX_UBUNTU)/tmp/apt sudo mkdir -p $(SANDBOX_UBUNTU)/tmp/apt
echo "Copying local ubuntu mirror into $(SANDBOX_UBUNTU)/tmp/apt" echo "Bind mounting local Ubuntu mirror to $(SANDBOX_UBUNTU)/tmp/apt"
sudo cp -al $(LOCAL_MIRROR)/ubuntu/dists $(LOCAL_MIRROR)/ubuntu/pool $(SANDBOX_UBUNTU)/tmp/apt sudo mount -o bind $(LOCAL_MIRROR)/ubuntu $(SANDBOX_UBUNTU)/tmp/apt
sudo mount -o remount,ro,bind $(SANDBOX_UBUNTU)/tmp/apt
echo "Configuring apt sources.list: deb file:///tmp/apt $(UBUNTU_RELEASE) main" echo "Configuring apt sources.list: deb file:///tmp/apt $(UBUNTU_RELEASE) main"
echo "deb file:///tmp/apt $(UBUNTU_RELEASE) main" | sudo tee $(SANDBOX_UBUNTU)/etc/apt/sources.list echo "deb file:///tmp/apt $(UBUNTU_RELEASE) main" | sudo tee $(SANDBOX_UBUNTU)/etc/apt/sources.list
echo "Allowing using unsigned repos" echo "Allowing using unsigned repos"
@ -85,6 +85,6 @@ echo "SANDBOX_UBUNTU_UP: done"
endef endef
define SANDBOX_UBUNTU_DOWN define SANDBOX_UBUNTU_DOWN
sync if mountpoint -q $(SANDBOX_UBUNTU)/proc; then sudo umount $(SANDBOX_UBUNTU)/proc; fi
sudo umount $(SANDBOX_UBUNTU)/proc sudo umount $(SANDBOX_UBUNTU)/tmp/apt || true
endef endef