openstack/fuel-web
Code Issues Proposed changes

make iso caching for comps.xml, ostf src zips

Browse Source 
Download comps.xml into a different directory than repodata so that
createrepo doesn't delete it after it's done.

Use separate target for each OSTF source zip so that it's only
re-downloaded from Github when commit SHA changes, and not
unconditionally downloaded for every build.
This commit is contained in:
Dmitry Borodaenko 2013-08-26 16:13:22 -07:00 committed by default
parent 51b872c140
commit b6e49c2b80
5 changed files with 18 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
iso/module.mk
View File

@ -12,7 +12,7 @@ $(BUILD_DIR)/iso/isoroot-centos.done: \
$(BUILD_DIR)/iso/isoroot-dotfiles.done $(BUILD_DIR)/iso/isoroot-dotfiles.done
mkdir -p $(ISOROOT) mkdir -p $(ISOROOT)
rsync -rp $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/ $(ISOROOT) rsync -rp $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/ $(ISOROOT)
createrepo -g $(ISOROOT)/repodata/comps.xml -x 'rhel/*' \ createrepo -g $(ISOROOT)/comps.xml -x 'rhel/*' \
-u media://`head -1 $(ISOROOT)/.discinfo` $(ISOROOT) -u media://`head -1 $(ISOROOT)/.discinfo` $(ISOROOT)
$(ACTION.TOUCH) $(ACTION.TOUCH)

8
mirror/centos/repo.mk
View File

@ -52,9 +52,9 @@ show-yum-urls-centos: \
--destdir=$(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages \ --destdir=$(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages \
$(REQUIRED_RPMS) $(REQUIRED_RPMS)
$(LOCAL_MIRROR_CENTOS_OS_BASEURL)/repodata/comps.xml: \ $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml: \
export COMPSXML=$(shell wget -qO- $(MIRROR_CENTOS_OS_BASEURL)/repodata/repomd.xml | grep -m 1 '$(@F)' | awk -F'"' '{ print $$2 }') export COMPSXML=$(shell wget -qO- $(MIRROR_CENTOS_OS_BASEURL)/repodata/repomd.xml | grep -m 1 '$(@F)' | awk -F'"' '{ print $$2 }')
$(LOCAL_MIRROR_CENTOS_OS_BASEURL)/repodata/comps.xml: $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml:
@mkdir -p $(@D) @mkdir -p $(@D)
if ( echo $${COMPSXML} | grep -q '\.gz$$' ); then \ if ( echo $${COMPSXML} | grep -q '\.gz$$' ); then \
wget -O $@.gz $(MIRROR_CENTOS_OS_BASEURL)/$${COMPSXML}; \ wget -O $@.gz $(MIRROR_CENTOS_OS_BASEURL)/$${COMPSXML}; \
@ -65,7 +65,7 @@ $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/repodata/comps.xml:
$(BUILD_DIR)/mirror/centos/repo.done: \ $(BUILD_DIR)/mirror/centos/repo.done: \
$(BUILD_DIR)/mirror/centos/yum.done \ $(BUILD_DIR)/mirror/centos/yum.done \
| $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/repodata/comps.xml | $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml
createrepo -g $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/repodata/comps.xml \ createrepo -g $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml \
-o $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/ $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/ -o $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/ $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/
$(ACTION.TOUCH) $(ACTION.TOUCH)

8
mirror/rhel/repo.mk
View File

@ -52,9 +52,9 @@ show-yum-urls-rhel: \
--destdir=$(LOCAL_MIRROR_RHEL)/Packages \ --destdir=$(LOCAL_MIRROR_RHEL)/Packages \
`echo $(REQ_RHEL_RPMS) | /bin/sed 's/-[0-9][0-9\.a-zA-Z_-]\+//g'` `echo $(REQ_RHEL_RPMS) | /bin/sed 's/-[0-9][0-9\.a-zA-Z_-]\+//g'`
$(LOCAL_MIRROR_RHEL)/repodata/comps.xml: \ $(LOCAL_MIRROR_RHEL)/comps.xml: \
export COMPSXML=$(shell wget -qO- $(MIRROR_RHEL)/repodata/repomd.xml | grep -m 1 '$(@F)' | awk -F'"' '{ print $$2 }') export COMPSXML=$(shell wget -qO- $(MIRROR_RHEL)/repodata/repomd.xml | grep -m 1 '$(@F)' | awk -F'"' '{ print $$2 }')
$(LOCAL_MIRROR_RHEL)/repodata/comps.xml: $(LOCAL_MIRROR_RHEL)/comps.xml:
@mkdir -p $(@D) @mkdir -p $(@D)
if ( echo $${COMPSXML} | grep -q '\.gz$$' ); then \ if ( echo $${COMPSXML} | grep -q '\.gz$$' ); then \
wget -O $@.gz $(MIRROR_RHEL)/$${COMPSXML}; \ wget -O $@.gz $(MIRROR_RHEL)/$${COMPSXML}; \
@ -81,7 +81,7 @@ $(BUILD_DIR)/mirror/rhel/fuel.done:
$(BUILD_DIR)/mirror/rhel/repo.done: \ $(BUILD_DIR)/mirror/rhel/repo.done: \
$(BUILD_DIR)/mirror/rhel/yum.done \ $(BUILD_DIR)/mirror/rhel/yum.done \
$(BUILD_DIR)/mirror/rhel/fuel.done \ $(BUILD_DIR)/mirror/rhel/fuel.done \
| $(LOCAL_MIRROR_RHEL)/repodata/comps.xml | $(LOCAL_MIRROR_RHEL)/comps.xml
createrepo -g $(LOCAL_MIRROR_RHEL)/repodata/comps.xml \ createrepo -g $(LOCAL_MIRROR_RHEL)/comps.xml \
-o $(LOCAL_MIRROR_RHEL)/ $(LOCAL_MIRROR_RHEL)/ -o $(LOCAL_MIRROR_RHEL)/ $(LOCAL_MIRROR_RHEL)/
$(ACTION.TOUCH) $(ACTION.TOUCH)

12
mirror/src/module.mk
View File

@ -7,14 +7,16 @@ else
endif endif
# #
# Download ostf packages directry from the github, because # Download ostf packages directly from the github, because
# it updates often and we don't want to update main mirrors # it updates often and we don't want to update main mirrors
# on each commit # on each commit
# #
$(BUILD_DIR)/mirror/src/build.done: $(call depv,OSTF_TESTS_SHA) $(LOCAL_MIRROR_SRC)/$(OSTF_TESTS_SHA).zip:
$(BUILD_DIR)/mirror/src/build.done: $(call depv,OSTF_PLUGIN_SHA)
$(BUILD_DIR)/mirror/src/build.done: $(SOURCE_DIR)/requirements-src.txt \
| $(addprefix $(LOCAL_MIRROR_SRC)/, $(notdir $(REQUIRED_SRCS)))
wget --no-use-server-timestamps -c -P $(LOCAL_MIRROR_SRC) https://github.com/Mirantis/fuel-ostf-tests/archive/$(OSTF_TESTS_SHA).zip wget --no-use-server-timestamps -c -P $(LOCAL_MIRROR_SRC) https://github.com/Mirantis/fuel-ostf-tests/archive/$(OSTF_TESTS_SHA).zip
$(LOCAL_MIRROR_SRC)/$(OSTF_PLUGIN_SHA).zip:
wget --no-use-server-timestamps -c -P $(LOCAL_MIRROR_SRC) https://github.com/Mirantis/fuel-ostf-plugin/archive/$(OSTF_PLUGIN_SHA).zip wget --no-use-server-timestamps -c -P $(LOCAL_MIRROR_SRC) https://github.com/Mirantis/fuel-ostf-plugin/archive/$(OSTF_PLUGIN_SHA).zip
$(BUILD_DIR)/mirror/src/build.done: $(SOURCE_DIR)/requirements-src.txt \
| $(addprefix $(LOCAL_MIRROR_SRC)/, $(notdir $(REQUIRED_SRCS)) $(OSTF_TESTS_SHA).zip $(OSTF_PLUGIN_SHA).zip)
$(ACTION.TOUCH) $(ACTION.TOUCH)

4
packages/rpm/module.mk
View File

@ -102,11 +102,11 @@ $(BUILD_DIR)/packages/rpm/repo.done: \
$(BUILD_DIR)/packages/rpm/rpm-rbenv-ruby.done \ $(BUILD_DIR)/packages/rpm/rpm-rbenv-ruby.done \
$(BUILD_DIR)/packages/rpm/rpm-mcollective.done $(BUILD_DIR)/packages/rpm/rpm-mcollective.done
find $(BUILD_DIR)/packages/rpm/RPMS -name '*.rpm' -exec cp -u {} $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages \; find $(BUILD_DIR)/packages/rpm/RPMS -name '*.rpm' -exec cp -u {} $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages \;
createrepo -g $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/repodata/comps.xml \ createrepo -g $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml \
-o $(LOCAL_MIRROR_CENTOS_OS_BASEURL) $(LOCAL_MIRROR_CENTOS_OS_BASEURL) -o $(LOCAL_MIRROR_CENTOS_OS_BASEURL) $(LOCAL_MIRROR_CENTOS_OS_BASEURL)
ifeq ($(CACHE_RHEL),1) ifeq ($(CACHE_RHEL),1)
find $(BUILD_DIR)/packages/rpm/RPMS -name '*.rpm' -exec cp -u {} $(LOCAL_MIRROR_RHEL)/Packages \; find $(BUILD_DIR)/packages/rpm/RPMS -name '*.rpm' -exec cp -u {} $(LOCAL_MIRROR_RHEL)/Packages \;
createrepo -g $(LOCAL_MIRROR_RHEL)/repodata/comps.xml \ createrepo -g $(LOCAL_MIRROR_RHEL)/comps.xml \
-o $(LOCAL_MIRROR_RHEL) $(LOCAL_MIRROR_RHEL) -o $(LOCAL_MIRROR_RHEL) $(LOCAL_MIRROR_RHEL)
endif endif
$(ACTION.TOUCH) $(ACTION.TOUCH)