Fix BUILD_PACKAGES flag

Build iso with BUILD_PACKAGES=0:

* Late packages:
  - fuel-bootstrap-image,
  - fuel-docker-images,
  should be always built in spite of  BUILD_PACKAGES flag,
  since they depend on the first level artifacts (packages)
  and are not updated in the mirrors yet;

* Build fuel-bootstrap-image-builder package for astute container

* Update requirements-*.txt file by adding all fuel packages only in case
  BUILD_PACKAGES=0

* Update docker-images spec to support fakeroot rpmbuild approach

Related-bug: #1460480

Change-Id: I00ea15c9df9277d48a294f8379d87352c017becb

mirror/centos/repo.mk

@@ -67,8 +67,20 @@ $(BUILD_DIR)/mirror/centos/rpm-download.done: $(BUILD_DIR)/mirror/centos/urls.li
 	xargs -n1 -P4 wget -Nnv -P "$$dst" < $<
 	$(ACTION.TOUCH)
 
+# BUILD_PACKAGES=0 - apply patch for requiremetns rpm, since we need fuel-packages
+ifeq ($(BUILD_PACKAGES),0)
+$(BUILD_DIR)/requirements-rpm.txt: \
+		$(SOURCE_DIR)/requirements-rpm.txt \
+		$(SOURCE_DIR)/requirements-fuel-rpm.txt
+	cat $^ | sort -u > $@.tmp
+	mv $@.tmp $@
+else
+$(BUILD_DIR)/requirements-rpm.txt: $(SOURCE_DIR)/requirements-rpm.txt
+	$(ACTION.COPY)
+endif
+
 # Strip the comments and sort the list alphabetically
-$(BUILD_DIR)/mirror/centos/requirements-rpm-0.txt: $(SOURCE_DIR)/requirements-rpm.txt
+$(BUILD_DIR)/mirror/centos/requirements-rpm-0.txt: $(BUILD_DIR)/requirements-rpm.txt
 	mkdir -p $(@D) && \
 	grep -v -e '^#' $< > $@.tmp && \
 	sort -u < $@.tmp > $@.pre && \

packages/module.mk

@@ -122,13 +122,9 @@ $(eval $(call prepare_file_source,fuel-bootstrap-image,bootstrap.rsa,$(SOURCE_DI
 
 .PHONY: packages-late packages-rpm-late
 
-ifneq ($(BUILD_PACKAGES),0)
 $(BUILD_DIR)/packages/build-late.done: \
 		$(BUILD_DIR)/packages/rpm/build-late.done \
 		$(BUILD_DIR)/packages/rpm/fuel-docker-images.done
-endif
-
-$(BUILD_DIR)/packages/build-late.done:
 	$(ACTION.TOUCH)
 
 packages-late: $(BUILD_DIR)/packages/build-late.done

packages/rpm/module.mk

@@ -101,6 +101,7 @@ $(BUILD_DIR)/packages/rpm/repo.done:
 	$(ACTION.TOUCH)
 
 $(BUILD_DIR)/packages/rpm/fuel-docker-images.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX/fuel-docker-images
+$(BUILD_DIR)/packages/rpm/fuel-docker-images.done: SPECFILE:=$(SOURCE_DIR)/packages/rpm/specs/fuel-docker-images.spec
 $(BUILD_DIR)/packages/rpm/fuel-docker-images.done: export SANDBOX_DOWN:=$(SANDBOX_DOWN)
 
 $(BUILD_DIR)/packages/rpm/fuel-docker-images.done: \
@@ -108,13 +109,14 @@ $(BUILD_DIR)/packages/rpm/fuel-docker-images.done: \
 		$(BUILD_DIR)/packages/rpm/buildd.tar.gz \
 		$(BUILD_DIR)/packages/rpm/repo-late.done \
 		$(BUILD_DIR)/docker/build.done
+	python $(SOURCE_DIR)/packages/rpm/genpkgnames.py $(SPECFILE) | xargs -I{} sudo find $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages -regex '.*/{}-[^-]+-[^-]+' -delete
 	mkdir -p $(BUILD_DIR)/packages/rpm/RPMS/x86_64
 	mkdir -p $(SANDBOX) && \
 	sudo tar xzf $(BUILD_DIR)/packages/rpm/buildd.tar.gz -C $(SANDBOX) && \
 	mkdir -p $(SANDBOX)/tmp/SOURCES && \
 	sudo cp -r $(BUILD_DIR)/docker/$(DOCKER_ART_NAME) $(SANDBOX)/tmp/SOURCES && \
 	(cd $(BUILD_DIR)/docker && sudo tar czf $(SANDBOX)/tmp/SOURCES/fuel-images-sources.tar.gz sources utils) && \
-	sudo cp $(SOURCE_DIR)/packages/rpm/specs/fuel-docker-images.spec $(SANDBOX)/tmp && \
+	sudo cp $(SPECFILE) $(SANDBOX)/tmp && \
 	sudo chroot $(SANDBOX) rpmbuild --nodeps --define "_topdir /tmp" -ba /tmp/fuel-docker-images.spec
 	cp $(SANDBOX)/tmp/RPMS/*/fuel-docker-images-*.rpm $(BUILD_DIR)/packages/rpm/RPMS/x86_64
 	find $(BUILD_DIR)/packages/rpm/RPMS -name '*.rpm' | xargs cp -u --target-directory=$(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages
@@ -122,9 +124,19 @@ $(BUILD_DIR)/packages/rpm/fuel-docker-images.done: \
 		-o $(LOCAL_MIRROR_CENTOS_OS_BASEURL) $(LOCAL_MIRROR_CENTOS_OS_BASEURL)
 	$(ACTION.TOUCH)
 
+# in case BUILD_PACKAGES=0 we have to build only fuel-bootstrap-image-builder
+ifeq (1,$(strip $(BUILD_PACKAGES)))
 $(BUILD_DIR)/packages/rpm/build.done: $(BUILD_DIR)/packages/rpm/repo.done
+else
+$(BUILD_DIR)/packages/rpm/build.done: $(BUILD_DIR)/bootstrap/fuel-bootstrap-image-builder-rpm.done \
+		$(BUILD_DIR)/mirror/centos/repo.done
+	find $(BUILD_DIR)/packages/rpm/RPMS -name '*.rpm' -exec cp -u {} $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages \;
+	createrepo -g $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml \
+	-o $(LOCAL_MIRROR_CENTOS_OS_BASEURL) $(LOCAL_MIRROR_CENTOS_OS_BASEURL)
+endif
 	$(ACTION.TOUCH)
 
+
 #######################################
 # This section is for building container
 # packages that depend on other packages.
@@ -144,7 +156,12 @@ fuel-bootstrap-image
 $(eval $(foreach pkg,$(fuel_rpm_packages_late),$(call build_rpm,$(pkg),-late)$(NEWLINE)))
 
 $(BUILD_DIR)/packages/rpm/repo.done: $(BUILD_DIR)/bootstrap/fuel-bootstrap-image-builder-rpm.done
+# BUILD_PACKAGES=0 - for late packages we need to be sure that centos mirror is ready
+# BUILD_PACKAGES=1 - for late packages we need to be sure that fuel-* packages was build beforehand
+$(BUILD_DIR)/packages/rpm/repo-late.done: $(BUILD_DIR)/mirror/centos/repo.done
+ifeq (1,$(strip $(BUILD_PACKAGES)))
 $(BUILD_DIR)/packages/rpm/repo-late.done: $(BUILD_DIR)/packages/rpm/repo.done
+endif
 	find $(BUILD_DIR)/packages/rpm/RPMS -name '*.rpm' -exec cp -u --target-directory $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages {} +
 	createrepo -g $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml \
 		-o $(LOCAL_MIRROR_CENTOS_OS_BASEURL) $(LOCAL_MIRROR_CENTOS_OS_BASEURL)

packages/rpm/specs/fuel-docker-images.spec

@@ -24,7 +24,9 @@ tar xzvf %{SOURCE1} -C %{name}-%{version}
 
 %install
 cd %{name}-%{version}
-mkdir -p %{buildroot}/var/www/nailgun/docker/{images,sources,utils}
+mkdir -p %{buildroot}/var/www/nailgun/docker/images
+mkdir -p %{buildroot}/var/www/nailgun/docker/sources
+mkdir -p %{buildroot}/var/www/nailgun/docker/utils
 install -m 644 %{SOURCE0} %{buildroot}/var/www/nailgun/docker/images/fuel-images.tar.lrz
 cp -R sources %{buildroot}/var/www/nailgun/docker/
 

requirements-fuel-rpm.txt

@@ -0,0 +1,26 @@
+fencing-agent
+fuel
+fuel-agent
+fuel-createmirror
+fuel-dockerctl
+fuel-ha-utils
+fuel-library
+fuelmenu
+fuel-migrate
+fuel-misc
+fuel-nailgun
+fuel-notify
+fuel-openstack-metadata
+fuel-ostf
+fuel-package-updates
+fuel-provisioning-scripts
+fuel-rabbit-fence
+fuel-release
+ironic-fa-bootstrap-configs
+nailgun-agent
+nailgun-mcagents
+network-checker
+python-fuelclient
+ruby21-nailgun-mcagents
+ruby21-rubygem-astute
+shotgun