diff --git a/config/common.cfg b/config/common.cfg index 50d098c..910cb1a 100644 --- a/config/common.cfg +++ b/config/common.cfg @@ -1,37 +1,58 @@ -VERSION="6.1" - -# Base directory for mirror +# Base directory for mirror MIRROR_ROOT="/var/www/nailgun" -# Log dir -LOG_ROOT="/var/log/mirror-sync" +# Configure upstream mirrors +# NOTE: only rsync mirrors are supported by the script! +# Please refer to https://launchpad.net/ubuntu/+archivemirrors for +# official upstream Ubuntu mirrors list +# +# Format: +# MIRROR_{DISTRO}_HOST - mirror hostname, i.e. "archive.ubuntu.com" +# MIRROR_{DISTRO}_DIR - source directory (rsync module name), i.e. "ubuntu" +# MIRROR_{DISTRO}_HTTPDIR - HTTP URL of mirror (usually equals to MIRROR_{DISTRO}_DIR) +# {DISTRO} - could be one of: UBUNTU, MOS_UBUNTU -# In case of working behind http proxy uncomment the following parameters -# Note that your proxy configuration must allow proxying to port 873 +# Upstream Ubuntu packages +MIRROR_UBUNTU_HOST="archive.ubuntu.com" +MIRROR_UBUNTU_DIR="ubuntu" +MIRROR_UBUNTU_HTTPDIR=$MIRROR_UBUNTU_DIR + +# MOS Ubuntu packages +MIRROR_MOS_UBUNTU_HOST="mirror.fuel-infra.org" +MIRROR_MOS_UBUNTU_DIR="mirror/mos/ubuntu" +MIRROR_MOS_UBUNTU_HTTPDIR="mos/ubuntu" + +# Configure repository paths +# Format: +# {DISTRO}_PATH=/path +# {DISTRO} - could be one of: UBUNTU, MOS_UBUNTU + +UBUNTU_PATH="${MIRROR_ROOT}/ubuntu-part" +MOS_UBUNTU_PATH="${MIRROR_ROOT}/mos-ubuntu" + +# Configure upstream packages mirroring mode +# PARTIAL_UPSTREAM: +# 0 - script will mirror all packages from specified distibutions +# and components. Upstream mirror structure will be preserved. +# 1 - (default) script will download only packages required for +# MOS. For Ubuntu packages script will create partial repository +# with the "main" component only, original mirror structure will +# not be preserved. +# NOTE: This setting affects upstream OS only (MOS mirror is always fetched as-is). + +PARTIAL_UPSTREAM=1 + +# In case of working behind http proxy uncomment the following parameters +# Note that your proxy configuration must allow proxying to port 873 # RSYNC_PROXY=username:password@host:port # export RSYNC_PROXY # http_proxy=http://username:password@host:port/ # export http_proxy -# Attempt to autodetect current Fuel version -if hash fuel 2>/dev/null; then - # working on Fuel node - FUEL_VERSION=`fuel --fuel-version 2>&1 | awk -v sq="'" '/^release:/ { gsub(sq,""); print $2 }'` - if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_VERSION}/config.mk -O /dev/null 2>/dev/null; then - FUEL_BRANCH=master - else - FUEL_BRANCH=$FUEL_VERSION - fi - # On Fuel node we will be using dockerized Ubuntu for dependencies resolving. - # Assuming that Docker is up and running. - DOCKER_MODE=true - FUEL_SERVER=`grep '"ipaddress":' /etc/fuel/astute.yaml | awk '{print $2}' | sed 's/"//g'` - FUEL_ADMIN_PASS=`grep -A 2 FUEL_ACCESS /etc/fuel/astute.yaml | grep password | awk '{print $2}' | sed 's/"//g'` - FULL_RELEASE=`fuel --fuel-version 2>&1 | grep -A 1 release_versions | tail -1 | cut -d":" -f1` -else - # working outside Fuel master node, using defaults - FUEL_VERSION=6.1 - FUEL_BRANCH=master - DOCKER_MODE=false - FUEL_SERVER=FUEL_SERVER -fi \ No newline at end of file +# On Fuel node (which is running CentOS) we use dockerized Ubuntu for dependencies resolving. +# Set this to "true" only if Docker is up and running. +# If set to "false", generic apt will be used instead. +DOCKER_MODE=true + +# Log dir +LOG_ROOT="/var/log/mirror-sync" diff --git a/config/config.mk b/config/config.mk new file mode 100644 index 0000000..f373203 --- /dev/null +++ b/config/config.mk @@ -0,0 +1,296 @@ +# +# Build directives. Can be overrided by environment variables. +# + +# Base path for build and mirror directories. +# Default value: current directory +TOP_DIR?=$(PWD) +TOP_DIR:=$(abspath $(TOP_DIR)) +# Working build directory +BUILD_DIR?=$(TOP_DIR)/build +BUILD_DIR:=$(abspath $(BUILD_DIR)) +# Path for build artifacts +ARTS_DIR?=$(BUILD_DIR)/artifacts +ARTS_DIR:=$(abspath $(ARTS_DIR)) +# Path for cache of downloaded packages +LOCAL_MIRROR?=$(TOP_DIR)/local_mirror +LOCAL_MIRROR:=$(abspath $(LOCAL_MIRROR)) +# Path to pre-built artifacts +DEPS_DIR?=$(TOP_DIR)/deps +DEPS_DIR:=$(abspath $(DEPS_DIR)) + +PRODUCT_VERSION:=6.1 + +# This variable is used for naming of auxillary objects +# related to product: repositories, mirrors etc +PRODUCT_NAME:=mos + +# This variable is used mostly for +# keeping things uniform. Some files +# contain versions as a part of their paths +# but building process for current version differs from +# ones for other versions which are supposed +# to come from DEPS_DIR "as is" +CURRENT_VERSION:=$(PRODUCT_VERSION) + +PACKAGE_VERSION=6.1.0 +UPGRADE_VERSIONS?=$(CURRENT_VERSION) + +# Path to pre-built artifacts +DEPS_DIR_CURRENT?=$(DEPS_DIR)/$(CURRENT_VERSION) +DEPS_DIR_CURRENT:=$(abspath $(DEPS_DIR_CURRENT)) + +# Artifacts names +ISO_NAME?=fuel-$(PRODUCT_VERSION) +UPGRADE_TARBALL_NAME?=fuel-$(PRODUCT_VERSION)-upgrade +OPENSTACK_PATCH_TARBALL_NAME?=fuel-$(PRODUCT_VERSION)-patch +VBOX_SCRIPTS_NAME?=vbox-scripts-$(PRODUCT_VERSION) +BOOTSTRAP_ART_NAME?=bootstrap.tar.gz +DOCKER_ART_NAME?=fuel-images.tar.lrz +VERSION_YAML_ART_NAME?=version.yaml +CENTOS_REPO_ART_NAME?=centos-repo.tar +UBUNTU_REPO_ART_NAME?=ubuntu-repo.tar +PUPPET_ART_NAME?=puppet.tgz +OPENSTACK_YAML_ART_NAME?=openstack.yaml +TARGET_UBUNTU_IMG_ART_NAME?=ubuntu_target_images.tar +TARGET_CENTOS_IMG_ART_NAME?=centos_target_images.tar + + + +# Where we put artifacts +ISO_PATH:=$(ARTS_DIR)/$(ISO_NAME).iso +IMG_PATH:=$(ARTS_DIR)/$(ISO_NAME).img +UPGRADE_TARBALL_PATH:=$(ARTS_DIR)/$(UPGRADE_TARBALL_NAME).tar +VBOX_SCRIPTS_PATH:=$(ARTS_DIR)/$(VBOX_SCRIPTS_NAME).zip + +MASTER_IP?=10.20.0.2 +MASTER_DNS?=10.20.0.1 +MASTER_NETMASK?=255.255.255.0 +MASTER_GW?=10.20.0.1 + +CENTOS_MAJOR:=6 +CENTOS_MINOR:=5 +CENTOS_RELEASE:=$(CENTOS_MAJOR).$(CENTOS_MINOR) +CENTOS_ARCH:=x86_64 +CENTOS_IMAGE_RELEASE:=$(CENTOS_MAJOR)$(CENTOS_MINOR) +UBUNTU_RELEASE:=trusty +UBUNTU_MAJOR:=14 +UBUNTU_MINOR:=04 +UBUNTU_RELEASE_NUMBER:=$(UBUNTU_MAJOR).$(UBUNTU_MINOR) +UBUNTU_KERNEL_FLAVOR?=lts-trusty +UBUNTU_NETBOOT_FLAVOR?=netboot +UBUNTU_ARCH:=amd64 +UBUNTU_IMAGE_RELEASE:=$(UBUNTU_MAJOR)$(UBUNTU_MINOR) +SEPARATE_IMAGES?=/boot,ext2 /,ext4 + +# Rebuld packages locally (do not use upstream versions) +BUILD_PACKAGES?=1 + +# Build OpenStack packages from external sources (do not use prepackaged versions) +# Enter the comma-separated list of OpenStack packages to build, or '0' otherwise. +# Example: BUILD_OPENSTACK_PACKAGES=neutron,keystone +BUILD_OPENSTACK_PACKAGES?=0 + +# Do not compress javascript and css files +NO_UI_OPTIMIZE:=0 + +# Define a set of defaults for each OpenStack package +# For each component defined in BUILD_OPENSTACK_PACKAGES variable, this routine will set +# the following variables (i.e. for 'BUILD_OPENSTACK_PACKAGES=neutron'): +# NEUTRON_REPO, NEUTRON_COMMIT, NEUTRON_SPEC_REPO, NEUTRON_SPEC_COMMIT, +# NEUTRON_GERRIT_URL, NEUTRON_GERRIT_COMMIT, NEUTRON_GERRIT_URL, +# NEUTRON_SPEC_GERRIT_URL, NEUTRON_SPEC_GERRIT_COMMIT +define set_vars + $(call uc,$(1))_REPO?=https://github.com/openstack/$(1).git + $(call uc,$(1))_COMMIT?=master + $(call uc,$(1))_SPEC_REPO?=https://review.fuel-infra.org/openstack-build/$(1)-build.git + $(call uc,$(1))_SPEC_COMMIT?=master + $(call uc,$(1))_GERRIT_URL?=https://review.openstack.org/openstack/$(1).git + $(call uc,$(1))_GERRIT_COMMIT?=none + $(call uc,$(1))_SPEC_GERRIT_URL?=https://review.fuel-infra.org/openstack-build/$(1)-build.git + $(call uc,$(1))_SPEC_GERRIT_COMMIT?=none +endef + +# Repos and versions +FUELLIB_COMMIT?=master +NAILGUN_COMMIT?=master +PYTHON_FUELCLIENT_COMMIT?=master +ASTUTE_COMMIT?=master +OSTF_COMMIT?=master + +FUELLIB_REPO?=https://github.com/stackforge/fuel-library.git +NAILGUN_REPO?=https://github.com/stackforge/fuel-web.git +PYTHON_FUELCLIENT_REPO?=https://github.com/stackforge/python-fuelclient.git +ASTUTE_REPO?=https://github.com/stackforge/fuel-astute.git +OSTF_REPO?=https://github.com/stackforge/fuel-ostf.git + +# Gerrit URLs and commits +FUELLIB_GERRIT_URL?=https://review.openstack.org/stackforge/fuel-library +NAILGUN_GERRIT_URL?=https://review.openstack.org/stackforge/fuel-web +PYTHON_FUELCLIENT_GERRIT_URL?=https://review.openstack.org/stackforge/python-fuelclient +ASTUTE_GERRIT_URL?=https://review.openstack.org/stackforge/fuel-astute +OSTF_GERRIT_URL?=https://review.openstack.org/stackforge/fuel-ostf + +FUELLIB_GERRIT_COMMIT?=none +NAILGUN_GERRIT_COMMIT?=none +PYTHON_FUELCLIENT_GERRIT_COMMIT?=none +ASTUTE_GERRIT_COMMIT?=none +OSTF_GERRIT_COMMIT?=none +FUELMAIN_GERRIT_COMMIT?=none + +LOCAL_MIRROR_CENTOS:=$(LOCAL_MIRROR)/centos +LOCAL_MIRROR_CENTOS_OS_BASEURL:=$(LOCAL_MIRROR_CENTOS)/os/$(CENTOS_ARCH) +LOCAL_MIRROR_UBUNTU:=$(LOCAL_MIRROR)/ubuntu +LOCAL_MIRROR_UBUNTU_OS_BASEURL:=$(LOCAL_MIRROR_UBUNTU) +LOCAL_MIRROR_DOCKER:=$(LOCAL_MIRROR)/docker +LOCAL_MIRROR_DOCKER_BASEURL:=$(LOCAL_MIRROR_DOCKER) + +# Use download.mirantis.com mirror by default. Other possible values are +# 'msk', 'srt', 'usa', 'hrk'. +# Setting any other value or removing of this variable will cause +# download of all the packages directly from internet +USE_MIRROR?=ext +ifeq ($(USE_MIRROR),ext) +YUM_REPOS?=proprietary +MIRROR_BASE?=http://mirror.fuel-infra.org/fwm/$(PRODUCT_VERSION) +MIRROR_CENTOS?=$(MIRROR_BASE)/centos +MIRROR_UBUNTU?=mirror.fuel-infra.org +MIRROR_UBUNTU_ROOT?=/$(PRODUCT_NAME)/ubuntu/ +MIRROR_UBUNTU_METHOD?=http +MIRROR_UBUNTU_SECTION?=main,restricted +MIRROR_DOCKER?=$(MIRROR_BASE)/docker +MIRROR_CENTOS_KERNEL?=$(MIRROR_CENTOS) +SANDBOX_MIRROR_CENTOS_UPSTREAM?=http://vault.centos.org/$(CENTOS_RELEASE) +endif +ifeq ($(USE_MIRROR),srt) +YUM_REPOS?=proprietary +MIRROR_BASE?=http://osci-mirror-srt.srt.mirantis.net/fwm/$(PRODUCT_VERSION) +MIRROR_CENTOS?=$(MIRROR_BASE)/centos +MIRROR_UBUNTU?=osci-mirror-srt.srt.mirantis.net +MIRROR_UBUNTU_ROOT?=/$(PRODUCT_NAME)/ubuntu/ +MIRROR_UBUNTU_METHOD?=http +MIRROR_UBUNTU_SECTION?=main,restricted +MIRROR_DOCKER?=$(MIRROR_BASE)/docker +MIRROR_CENTOS_KERNEL?=$(MIRROR_CENTOS) +endif +ifeq ($(USE_MIRROR),msk) +YUM_REPOS?=proprietary +MIRROR_BASE?=http://osci-mirror-msk.msk.mirantis.net/fwm/$(PRODUCT_VERSION) +MIRROR_CENTOS?=$(MIRROR_BASE)/centos +MIRROR_UBUNTU?=osci-mirror-msk.msk.mirantis.net +MIRROR_UBUNTU_ROOT?=/$(PRODUCT_NAME)/ubuntu/ +MIRROR_UBUNTU_METHOD?=http +MIRROR_UBUNTU_SECTION?=main,restricted +MIRROR_DOCKER?=$(MIRROR_BASE)/docker +MIRROR_CENTOS_KERNEL?=$(MIRROR_CENTOS) +endif +ifeq ($(USE_MIRROR),hrk) +YUM_REPOS?=proprietary +MIRROR_BASE?=http://osci-mirror-kha.kha.mirantis.net/fwm/$(PRODUCT_VERSION) +MIRROR_CENTOS?=$(MIRROR_BASE)/centos +MIRROR_UBUNTU?=osci-mirror-kha.kha.mirantis.net +MIRROR_UBUNTU_ROOT?=/$(PRODUCT_NAME)/ubuntu/ +MIRROR_UBUNTU_METHOD?=http +MIRROR_UBUNTU_SECTION?=main,restricted +MIRROR_DOCKER?=$(MIRROR_BASE)/docker +MIRROR_CENTOS_KERNEL?=$(MIRROR_CENTOS) +endif +ifeq ($(USE_MIRROR),usa) +YUM_REPOS?=proprietary +MIRROR_BASE?=http://mirror.seed-us1.fuel-infra.org/fwm/$(PRODUCT_VERSION) +MIRROR_CENTOS?=$(MIRROR_BASE)/centos +MIRROR_UBUNTU?=mirror.seed-us1.fuel-infra.org +MIRROR_UBUNTU_ROOT?=/$(PRODUCT_NAME)/ubuntu/ +MIRROR_UBUNTU_METHOD?=http +MIRROR_UBUNTU_SECTION?=main,restricted +MIRROR_DOCKER?=$(MIRROR_BASE)/docker +MIRROR_CENTOS_KERNEL?=$(MIRROR_CENTOS) +endif +ifeq ($(USE_MIRROR),cz) +YUM_REPOS?=proprietary +MIRROR_BASE?=http://mirror.seed-cz1.fuel-infra.org/fwm/$(PRODUCT_VERSION) +MIRROR_CENTOS?=$(MIRROR_BASE)/centos +MIRROR_UBUNTU?=mirror.seed-cz1.fuel-infra.org +MIRROR_UBUNTU_ROOT?=/$(PRODUCT_NAME)/ubuntu/ +MIRROR_UBUNTU_METHOD?=http +MIRROR_UBUNTU_SECTION?=main,restricted +MIRROR_DOCKER?=$(MIRROR_BASE)/docker +MIRROR_CENTOS_KERNEL?=$(MIRROR_CENTOS) +endif + + +#This suffix is used to generate path +#to ubuntu mirror inside mirror +#DocumentRoot + +MIRROR_UBUNTU_SUFFIX?=/pkgs/ubuntu + +YUM_DOWNLOAD_SRC?= + +MIRROR_CENTOS?=http://mirrors-local-msk.msk.mirantis.net/centos-$(PRODUCT_VERSION)/$(CENTOS_RELEASE) +MIRROR_CENTOS_KERNEL?=http://mirror.centos.org/centos-6/6.6/ +MIRROR_CENTOS_OS_BASEURL:=$(MIRROR_CENTOS)/os/$(CENTOS_ARCH) +MIRROR_CENTOS_KERNEL_BASEURL?=$(MIRROR_CENTOS_KERNEL)/os/$(CENTOS_ARCH) +MIRROR_UBUNTU?=osci-mirror-msk.msk.mirantis.net +MIRROR_UBUNTU_OS_BASEURL:=$(MIRROR_UBUNTU) +MIRROR_UBUNTU_METHOD?=http +MIRROR_UBUNTU_ROOT?=/$(PRODUCT_NAME)/ubuntu/ +MIRROR_UBUNTU_SECTION?=main,restricted +MIRROR_DOCKER?=http://mirror.fuel-infra.org/fwm/$(PRODUCT_VERSION)/docker +MIRROR_DOCKER_BASEURL:=$(MIRROR_DOCKER) +# MIRROR_FUEL option is valid only for 'fuel' YUM_REPOS section +# and ignored in other cases +MIRROR_POSTFIX?=stable +MIRROR_FUEL?=http://osci-obs.vm.mirantis.net:82/centos-fuel-$(PRODUCT_VERSION)-$(MIRROR_POSTFIX)/centos/ +ifeq (precise,$(strip $(UBUNTU_RELEASE))) +MIRROR_FUEL_UBUNTU?=http://osci-obs.vm.mirantis.net:82/ubuntu-fuel-$(PRODUCT_VERSION)-$(MIRROR_POSTFIX)/reprepro +else +MIRROR_FUEL_UBUNTU?=obs-1.mirantis.com +endif + +REQUIRED_RPMS:=$(shell grep -v "^\\s*\#" $(SOURCE_DIR)/requirements-rpm.txt) +REQUIRED_DEBS:=$(shell grep -v "^\\s*\#" $(SOURCE_DIR)/requirements-deb.txt) + +# Which repositories to use for making local centos mirror. +# Possible values you can find out from mirror/centos/yum_repos.mk file. +# The actual name will be constracted wich prepending "yum_repo_" prefix. +# Example: YUM_REPOS?=official epel => yum_repo_official and yum_repo_epel +# will be used. +YUM_REPOS?=official fuel subscr_manager + +# Additional CentOS repos. +# Each repo must be comma separated tuple with repo-name and repo-path. +# Repos must be separated by space. +# Example: EXTRA_RPM_REPOS="lolo,http://my.cool.repo/rpm bar,ftp://repo.foo" +EXTRA_RPM_REPOS?= + +# Comma or space separated list. Available feature groups: +# experimental - allow experimental options +# mirantis - enable Mirantis logos and support page +FEATURE_GROUPS?=experimental +comma:=, +FEATURE_GROUPS:=$(subst $(comma), ,$(FEATURE_GROUPS)) + +# INTEGRATION TEST CONFIG +NOFORWARD:=1 + +# Path to yaml configuration file to build ISO ks.cfg +KSYAML?=$(SOURCE_DIR)/iso/ks.yaml + +# Docker prebuilt containers. Default is to build containers during ISO build +DOCKER_PREBUILT?=0 + +# Source of docker prebuilt containers archive. Works only if DOCKER_PREBUILT=true +# Examples: +# DOCKER_PREBUILT_SOURCE=http://srv11-msk.msk.mirantis.net/docker-test/fuel-images.tar.lrz +# DOCKER_PREBUILT_SOURCE=/var/fuel-images.tar.lrz make docker +DOCKER_PREBUILT_SOURCE?=http://srv11-msk.msk.mirantis.net/docker-test/fuel-images.tar.lrz + +# Production variable (prod, dev, docker) +PRODUCTION?=docker + +SANDBOX_MIRROR_CENTOS_UPSTREAM?=http://mirrors-local-msk.msk.mirantis.net/centos-$(PRODUCT_VERSION)/$(CENTOS_RELEASE) +SANDBOX_MIRROR_CENTOS_UPSTREAM_OS_BASEURL:=$(SANDBOX_MIRROR_CENTOS_UPSTREAM)/os/$(CENTOS_ARCH)/ +SANDBOX_MIRROR_CENTOS_UPDATES_OS_BASEURL:=$(SANDBOX_MIRROR_CENTOS_UPSTREAM)/updates/$(CENTOS_ARCH)/ +SANDBOX_MIRROR_EPEL?=http://mirror.yandex.ru/epel/ +SANDBOX_MIRROR_EPEL_OS_BASEURL:=$(SANDBOX_MIRROR_EPEL)/$(CENTOS_MAJOR)/$(CENTOS_ARCH)/ diff --git a/config/fuel.cfg b/config/fuel.cfg new file mode 100644 index 0000000..ac2b2a8 --- /dev/null +++ b/config/fuel.cfg @@ -0,0 +1,9 @@ +# Get Fuel settings + +source <( $BINROOT/util/parse_yaml.py /etc/fuel/astute.yaml "FUEL" ) +source <( $BINROOT/util/parse_yaml.py /etc/nailgun/version.yaml "FUEL" ) + +FUEL_VERSION=$FUEL_VERSION_release +FUEL_SERVER=$FUEL_ADMIN_NETWORK_ipaddress +FUEL_ADMIN_PASS=$FUEL_FUEL_ACCESS_password +FULL_RELEASE=$FUEL_VERSION_openstack_version \ No newline at end of file diff --git a/config/mos-ubuntu-updatesonly.cfg b/config/mos-ubuntu-updatesonly.cfg index 0efe042..91a83f5 100644 --- a/config/mos-ubuntu-updatesonly.cfg +++ b/config/mos-ubuntu-updatesonly.cfg @@ -1,17 +1,17 @@ # Source common config source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/common.cfg" +source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/fuel.cfg" # Sync source -UPSTREAM="mirror.fuel-infra.org" -UPSTREAM_DIR="mirror/mos/ubuntu" +UPSTREAM=$MIRROR_MOS_UBUNTU_HOST +UPSTREAM_DIR=$MIRROR_UBUNTU_DIR UPSTREAM_DIR_HTTP="mos/ubuntu" -# PARTIAL_UPSTREAM=0 : sync full upstream repository -# PARTIAL_UPSTREAM=1 : sync only upstream packages required by MOS +# Always sync full MOS mirror PARTIAL_UPSTREAM=0 # Sync destination -LOCAL_DIR="${MIRROR_ROOT:-"/var/www/nailgun"}/mos-ubuntu" +LOCAL_DIR=$MOS_UBUNTU_PATH # What parts to sync I18Ns=( en ) # Translations, not delete english, it might break repo diff --git a/config/mos-ubuntu.cfg b/config/mos-ubuntu.cfg index 63830aa..2c2cd44 100644 --- a/config/mos-ubuntu.cfg +++ b/config/mos-ubuntu.cfg @@ -1,17 +1,17 @@ # Source common config source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/common.cfg" +source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/fuel.cfg" # Sync source -UPSTREAM="mirror.fuel-infra.org" -UPSTREAM_DIR="mirror/mos/ubuntu" +UPSTREAM=$MIRROR_MOS_UBUNTU_HOST +UPSTREAM_DIR=$MIRROR_UBUNTU_DIR UPSTREAM_DIR_HTTP="mos/ubuntu" -# PARTIAL_UPSTREAM=0 : sync full upstream repository -# PARTIAL_UPSTREAM=1 : sync only upstream packages required by MOS +# Always sync full MOS mirror PARTIAL_UPSTREAM=0 # Sync destination -LOCAL_DIR="${MIRROR_ROOT:-"/var/www/nailgun"}/mos-ubuntu" +LOCAL_DIR=$MOS_UBUNTU_PATH # What parts to sync I18Ns=( en ) # Translations, not delete english, it might break repo diff --git a/config/requirements-deb.txt b/config/requirements-deb.txt new file mode 100644 index 0000000..4c959d1 --- /dev/null +++ b/config/requirements-deb.txt @@ -0,0 +1,246 @@ +anacron +apache2 +aptitude +atop +bc +bridge-utils +build-essential +ceilometer-agent-central +ceilometer-agent-compute +ceilometer-agent-notification +ceilometer-alarm-evaluator +ceilometer-alarm-notifier +ceilometer-api +ceilometer-collector +ceilometer-common +ceph +ceph-deploy +ceph-mds +cinder-api +cinder-backup +cinder-scheduler +cinder-volume +cirros-testvm +cirros-testvmware +cloud-init +conntrackd +corosync +cpu-checker +cpufrequtils +curl +daemonize +debconf-utils +debhelper +devscripts +discover +discover-data +disktype +dkms +dnsmasq-base +dnsmasq-utils +dstat +erlang-dev +ethtool +fence-agents +fuel-utils +galera +git +glance-api +glance-common +glance-registry +grub-pc +haproxy +heat-api +heat-api-cfn +heat-api-cloudwatch +heat-common +heat-docker +heat-engine +hicolor-icon-theme +htop +ifenslave +installation-report +intel-microcode +iperf +iperf3 +ipmitool +iptables-persistent +kdump-tools +keystone +language-pack-en +language-pack-en-base +language-pack-gnome-en +language-pack-gnome-en-base +libapache2-mod-fastcgi +libapache2-mod-wsgi +libguestfs-tools +libnss3-tools +libssl0.9.8 +linux-firmware +linux-firmware-nonfree +lksctp-tools +lldpad +mc +mcelog +mcollective +mcollective-client +mdadm +memcached +mongodb-clients +mongodb-server +monit +murano +murano-dashboard +mysql-client +mysql-server-wsrep-5.6 +nailgun-mcagents +nailgun-net-check +neutron-dhcp-agent +neutron-l3-agent +neutron-lbaas-agent +neutron-metadata-agent +neutron-metering-agent +neutron-plugin-bigswitch +neutron-plugin-bigswitch-agent +neutron-plugin-brocade +neutron-plugin-cisco +neutron-plugin-hyperv +neutron-plugin-ibm +neutron-plugin-ibm-agent +neutron-plugin-linuxbridge +neutron-plugin-linuxbridge-agent +neutron-plugin-metaplugin +neutron-plugin-metering-agent +neutron-plugin-midonet +neutron-plugin-ml2 +neutron-plugin-mlnx +neutron-plugin-mlnx-agent +neutron-plugin-nec +neutron-plugin-nec-agent +neutron-plugin-nicira +neutron-plugin-oneconvergence +neutron-plugin-oneconvergence-agent +neutron-plugin-openflow-agent +neutron-plugin-openvswitch +neutron-plugin-openvswitch-agent +neutron-plugin-plumgrid +neutron-plugin-ryu +neutron-plugin-ryu-agent +neutron-plugin-vmware +neutron-plugin-vpn-agent +neutron-server +neutron-vpn-agent +nfs-common +nginx +nmap +nodejs +node-less +nova-api +nova-cert +nova-compute-kvm +nova-compute-qemu +nova-conductor +nova-consoleauth +nova-network +nova-novncproxy +nova-objectstore +nova-scheduler +novnc +ntp +ohai +openhpid +openssh-server +openstack-dashboard +openvswitch-switch +os-prober +pacemaker +pcs +percona-toolkit +percona-xtrabackup +pssh +puppet +python-carrot +python-ceilometer +python-ceilometerclient +python-ceph +python-daemon +python-daemonize +python-dingus +python-ethtool +python-feedparser +python-gflags +python-glance +python-gobject +python-gobject-2 +python-heatclient +python-keyring +python-keystone +python-ldap +python-libxml2 +python-memcache +python-muranoclient +python-mysqldb +python-netifaces +python-neutron +python-nose +python-novnc +python-openid +python-openstack-auth +python-openstackclient +python-passlib +python-pastescript +python-pexpect +python-pycurl +python-pylibmc +python-pyudev +python-rabbit +python-saharaclient +python-scapy +python-scgi +python-setuptools-git +python-sqlalchemy-ext +python-stompy +python-swift +python-utidylib +python-zmq +qemu-kvm +qemu-utils +rabbitmq-server +radosgw +ruby +ruby-cstruct +ruby-dev +ruby-httpclient +ruby-i18n +ruby-ipaddress +ruby-json +ruby-netaddr +ruby-openstack +ruby-rethtool +ruby-stomp +sahara +screen +sheepdog +socat +swift +swift-account +swift-container +swift-object +swift-plugin-s3 +swift-proxy +sysstat +tasksel +tcptraceroute +tmux +traceroute +ubuntu-minimal +ubuntu-standard +vgabios +vim +virt-what +vlan +xinetd +zabbix-agent +zabbix-frontend-php +zabbix-server-mysql +zabbix-server-pgsql diff --git a/config/ubuntu.cfg b/config/ubuntu.cfg index 5d11d4c..354d113 100644 --- a/config/ubuntu.cfg +++ b/config/ubuntu.cfg @@ -1,20 +1,21 @@ # Source common config source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/common.cfg" +source "$(dirname $(readlink -f "${BASH_SOURCE[0]}"))/fuel.cfg" # Sync source -UPSTREAM="archive.ubuntu.com" -UPSTREAM_DIR="ubuntu" -UPSTREAM_DIR_HTTP=${UPSTREAM_DIR} +UPSTREAM=$MIRROR_UBUNTU_HOST +UPSTREAM_DIR=$MIRROR_UBUNTU_DIR +UPSTREAM_DIR_HTTP=$MIRROR_UBUNTU_HTTPDIR + +PARTIAL_UPSTREAM=$PARTIAL_UPSTREAM # Sync destination -LOCAL_DIR="${MIRROR_ROOT:-"/var/www/nailgun"}/ubuntu-full" - -# PARTIAL_UPSTREAM=0 : sync full upstream repository -# PARTIAL_UPSTREAM=1 : sync only upstream packages required by MOS -PARTIAL_UPSTREAM=1 - -# Destionation for partial upstream mirror -PARTIAL_UPSTREAM_PATH="${MIRROR_ROOT:-"/var/www/mirror"}/ubuntu-part" +if [ "$PARTIAL_UPSTREAM" == "1" ]; then + LOCAL_DIR="${MIRROR_ROOT:-"/var/www/nailgun"}/ubuntu-full" + PARTIAL_UPSTREAM_PATH=$UBUNTU_PATH +else + LOCAL_DIR=$UBUNTU_PATH +fi # What parts to sync I18Ns=( en ) # Translations, not delete english, it might break repo diff --git a/deb-mirror b/deb-mirror index 2c4367c..ac8a348 100755 --- a/deb-mirror +++ b/deb-mirror @@ -457,24 +457,20 @@ if [[ "$PARTIAL_UPSTREAM" = "1" ]]; then info "Resolving dependencies for partial mirror" # Detect kernel version of debian-installer -fuel_config_temp=`mktemp --suffix="-fuel-config-temp"` -if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_BRANCH}/config.mk -O $fuel_config_temp; then - fatal "Failed to fetch Fuel config" -fi -export UBUNTU_RELEASE=`awk -F '[:=?]' '/^UBUNTU_RELEASE\>/ {print $NF}' $fuel_config_temp` -export UBUNTU_NETBOOT_FLAVOR=`awk -F '[:=?]' '/^UBUNTU_NETBOOT_FLAVOR\>/ {print $NF}' $fuel_config_temp` -export UBUNTU_ARCH=`awk -F '[:=?]' '/^UBUNTU_ARCH\>/ {print $NF}' $fuel_config_temp` +fuel_config=$BINROOT/config/config.mk +export UBUNTU_RELEASE=`awk -F '[:=?]' '/^UBUNTU_RELEASE\>/ {print $NF}' $fuel_config` +export UBUNTU_NETBOOT_FLAVOR=`awk -F '[:=?]' '/^UBUNTU_NETBOOT_FLAVOR\>/ {print $NF}' $fuel_config` +export UBUNTU_KERNEL_FLAVOR=`awk -F '[:=?]' '/^UBUNTU_KERNEL_FLAVOR\>/ {print $NF}' $fuel_config` +export UBUNTU_ARCH=`awk -F '[:=?]' '/^UBUNTU_ARCH\>/ {print $NF}' $fuel_config` INITRD_DIR="/dists/${UBUNTU_RELEASE}-updates/main/installer-${UBUNTU_ARCH}/current/images/${UBUNTU_NETBOOT_FLAVOR}/ubuntu-installer/${UBUNTU_ARCH}/" mkdir -p "$LOCAL_DIR/$INITRD_DIR" fetch "/$INITRD_DIR/initrd.gz" "$LOCAL_DIR/$INITRD_DIR" -UBUNTU_INSTALLER_KERNEL_VERSION=`zcat "$LOCAL_DIR/$INITRD_DIR/initrd.gz" | cpio --list 'lib/modules/*/kernel' 2>/dev/null | cut -d"/" -f 3` +export UBUNTU_INSTALLER_KERNEL_VERSION=`zcat "$LOCAL_DIR/$INITRD_DIR/initrd.gz" | cpio --list 'lib/modules/*/kernel' 2>/dev/null | cut -d"/" -f 3` debug "Detected debian-installer kernel version: "$UBUNTU_INSTALLER_KERNEL_VERSION -rm -f "$fuel_config_temp" # Generate list of MOS dependencies export apt_altstate=`mktemp -d --suffix="-apt-altstate"` -export FUEL_BRANCH export BINROOT if [[ "$DOCKER_MODE" = "true" ]]; then @@ -482,8 +478,9 @@ if [[ "$DOCKER_MODE" = "true" ]]; then # docker pull ubuntu:latest docker -D run -d --name=fuel-createmirror --net=host -a stdout -a stderr -t \ -e UBUNTU_RELEASE=$UBUNTU_RELEASE -e UBUNTU_NETBOOT_FLAVOR=$UBUNTU_NETBOOT_FLAVOR \ + -e UBUNTU_INSTALLER_KERNEL_VERSION=$UBUNTU_INSTALLER_KERNEL_VERSION -e UBUNTU_KERNEL_FLAVOR=$UBUNTU_KERNEL_FLAVOR \ -e RSYNC_PROXY=$RSYNC_PROXY -e http_proxy=$http_proxy \ - -e UBUNTU_ARCH=$UBUNTU_ARCH -e FUEL_BRANCH=$FUEL_BRANCH -e BINROOT=$BINROOT \ + -e UBUNTU_ARCH=$UBUNTU_ARCH -e BINROOT=$BINROOT \ -e apt_altstate=$apt_altstate -v $BINROOT:$BINROOT:rw -v $apt_altstate:$apt_altstate:rw ubuntu:latest dockerctl shell fuel-createmirror $BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies" # cleanup ubuntu container diff --git a/fuel-createmirror b/fuel-createmirror index 3ac61aa..9bcaa06 100755 --- a/fuel-createmirror +++ b/fuel-createmirror @@ -2,61 +2,51 @@ # This shell script was written in order to help you to create and maintain your # local mirrors of MOS and/or Ubuntu. You could use this script as a cron job. -# Dependencies: rsync, wget, gpg, docker + dpkg-dev (only for partial Ubuntu mirror) +# Dependencies: rsync, gpg, docker + dpkg-dev (only for partial Ubuntu mirror) -usage(){ +usage() { cat <&2 ; exit 1; } print_repositories_ubuntu() { +# $1 - directory name of local repository echo -e " * INFO: In order to setup these repositories MANUALLY, you should" echo -e " go to Fuel UI, choose your cluster and go to the 'Settings' tab" if [ "$PARTIAL_UPSTREAM" == "0" ]; then @@ -78,50 +68,97 @@ add_repositories_to_nailgun() { # $2 - distro name in fuel-package-updates format # $3 - directory name of local repository echo " * INFO: Attempting to add created repositories to Nailgun..." - # find envs with status "new" and with given release_id local release_id=$1 - local extraopts="" - [ "$PARTIAL_UPSTREAM" == "1" ] && extraopts=" --clear-upstream-repos " + local clearupstream=" --clear-upstream-repos " + local makedefault=" --make-default " + local apply=" --apply " + [ "$PARTIAL_UPSTREAM" == "0" ] && clearupstream="" + [ "$OPT_NO_DEFAULT" == "1" ] && makedefault="" + [ "$OPT_NO_APPLY" == "1" ] && apply="" + # find envs with status "new" and with given release_id envs=`fuel env 2>&1 | grep -w new | awk -v release_id=$release_id -F'|' '$5 == release_id {print $1}'` for env in ${envs}; do - fuel-package-updates -d $2 -r $FULL_RELEASE --no-download --apply --make-default \ - -s $FUEL_SERVER -p "$FUEL_ADMIN_PASS" -b http://$FUEL_SERVER:8080/$3 -e $env $extraopts 2>/dev/null + $EXEC_PREFIX fuel-package-updates -d $2 -r $FULL_RELEASE --no-download $makedefault $apply \ + -s $FUEL_SERVER -p "$FUEL_ADMIN_PASS" -b http://$FUEL_SERVER:8080/$3 -e $env $clearupstream 2>/dev/null EC_FPU=$? if [[ "$EC_FPU" == "0" ]]; then - echo " * INFO: environment id=$env updated successfully, no manual actions is required" + [ "$OPT_NO_APPLY" ] || echo " * INFO: environment id=$env updated successfully, no manual actions is required" else - echo " * WARNING: Failed to add repositories for environment id=$env to Nailgun, please add them MANUALLY" + echo " * INFO: Failed to add repositories for environment id=$env to Nailgun, please add them MANUALLY" EC_ADD=1 fi + [ "$OPT_NO_DEFAULT" ] && echo " * INFO: Default repositories for new environments were not modified" || echo " * INFO: Created repositories were set as defaults for new environments" done [ "$EC_ADD" == "1" ] && print_repositories_ubuntu $3 } +### BEGIN + +# Set defaults +OPT_MOS=1 +OPT_UBUNTU=1 +EXEC_PREFIX="" + +# Parse options +OPTS=`getopt -o hdaMUN -l help,no-default,no-apply,mos,ubuntu,dry-run -- "$@"` +if [ $? != 0 ]; then + usage_short + exit 1 +fi + +eval set -- "$OPTS" + +while true ; do + case "$1" in + -h| --help ) usage ; exit 0;; + -d | --no-default ) OPT_NO_DEFAULT=1; shift;; + -a | --no-apply ) OPT_NO_APPLY=1; shift;; + -N | --dry-run ) EXEC_PREFIX="echo EXEC "; shift;; + -M | --mos ) unset OPT_UBUNTU; shift;; + -U | --ubuntu ) unset OPT_MOS; shift;; + -- ) shift; break;; + * ) break;; + esac +done + +if [[ "$@" != "" ]]; then + echo "Invalid option -- $@" + usage_short + exit 1 +fi + +if [ -z $OPT_MOS ] && [ -z $OPT_UBUNTU ]; then + echo "The --mos and --ubuntu options are mutually exclusive, aborting..." + usage_short + exit 1 +fi + BINROOT=$(dirname `readlink -f "$0"`) -if [[ ( "$1" == "--help" ) || ( "$1" == "-h" ) ]]; then - usage - exit 0 -fi - . $BINROOT/config/common.cfg +. $BINROOT/config/fuel.cfg -mkdir -p ${MIRROR_ROOT} || die "Cannot create ${MIRROR_ROOT}, exiting." -mkdir -p ${LOG_ROOT} || die "Cannot create ${LOG_ROOT}, exiting." +if [ -z "${RSYNC_PROXY+x}" ] && [ $http_proxy ]; then + export http_proxy + export RSYNC_PROXY=$http_proxy +fi + +$EXEC_PREFIX mkdir -p ${MIRROR_ROOT} || die "Cannot create ${MIRROR_ROOT}, exiting." +$EXEC_PREFIX mkdir -p ${LOG_ROOT} || die "Cannot create ${LOG_ROOT}, exiting." EC=0 -if [[ "$1" != "ubuntu" ]]; then - $BINROOT/deb-mirror $BINROOT/config/mos-ubuntu-updatesonly.cfg +if [[ $OPT_MOS ]]; then + $EXEC_PREFIX $BINROOT/deb-mirror $BINROOT/config/mos-ubuntu-updatesonly.cfg EC_MOS=$? fi -if [[ "$1" != "mos" ]]; then - $BINROOT/deb-mirror $BINROOT/config/ubuntu.cfg +if [[ $OPT_UBUNTU ]]; then + $EXEC_PREFIX $BINROOT/deb-mirror $BINROOT/config/ubuntu.cfg EC_UBUNTU=$? fi -if [[ "$1" != "ubuntu" ]]; then +if [[ $OPT_MOS ]]; then if [[ "$EC_MOS" == "0" ]]; then . $BINROOT/config/mos-ubuntu-updatesonly.cfg echo " * INFO: MOS mirror was created at: $LOCAL_DIR" @@ -135,7 +172,7 @@ if [[ "$1" != "ubuntu" ]]; then EC=1 fi fi -if [[ "$1" != "mos" ]]; then +if [[ $OPT_UBUNTU ]]; then if [[ "$EC_UBUNTU" == "0" ]]; then . $BINROOT/config/ubuntu.cfg if [[ $PARTIAL_UPSTREAM = "1" ]]; then diff --git a/util/dpkg.sh b/util/dpkg.sh index 1941812..0dda3f9 100644 --- a/util/dpkg.sh +++ b/util/dpkg.sh @@ -153,24 +153,3 @@ pkg_file_valid() debug "md5sum = $md5sum, expected $expected_md5sum" return 1 } - -# DEPRECATED -parse_pkg_file() -{ - local file="$1" - local to_return=() - - local line - declare -A entry - - read_file "$file" | \ - while read line; do - if [[ -z "$line" ]]; then - continue - fi - param=${line%%:*} - entry[$param]=#${line#*: } - # #echo "${entry[md5sum]} ${entry[size]} ${entry[filename]}" - # entry=() - done -} diff --git a/util/parse_yaml.py b/util/parse_yaml.py new file mode 100755 index 0000000..42235dd --- /dev/null +++ b/util/parse_yaml.py @@ -0,0 +1,27 @@ +#!/usr/bin/python +# Enumerate YAML file and produce prefixed output + +import re +import sys +import yaml + +filename = sys.argv[1] +prefix = sys.argv[2] + +def serialize(value, name): + if value is None: + print('{0}=""'.format(name)) + elif hasattr(value, 'items'): + for key, subvalue in value.items(): + key = re.sub(r'[\W]', '_', key) + serialize(subvalue, name + '_' + key) + elif hasattr(value, '__iter__'): + print("{0}_len={1}".format(name, len(value))) + for i, v in enumerate(value): + serialize(v, name + '_' + str(i)) + else: + print('{0}="{1}"'.format(name, value)) + +with open(filename, 'r') as yaml_file: + data = yaml.load(yaml_file) + serialize(data, prefix) diff --git a/util/partial_ubuntu.sh b/util/partial_ubuntu.sh index 8ed6f18..89ffed9 100755 --- a/util/partial_ubuntu.sh +++ b/util/partial_ubuntu.sh @@ -1,9 +1,20 @@ #!/bin/bash -if [ -z "$FUEL_BRANCH" ]; then - echo "`basename $0`: FUEL_BRANCH is not defined!" - exit 1 -fi +cp $BINROOT/config/requirements-deb.txt $apt_altstate + +cat >> $apt_altstate/requirements-deb.txt << EOF +linux-image-${UBUNTU_INSTALLER_KERNEL_VERSION} +linux-headers-${UBUNTU_INSTALLER_KERNEL_VERSION} +linux-image-generic-${UBUNTU_KERNEL_FLAVOR} +linux-headers-generic-${UBUNTU_KERNEL_FLAVOR} +EOF + +requirements_add_essential_pkgs () { + # All essential packages are already installed, so ask dpkg for a list + dpkg-query -W -f='${Package} ${Essential}\n' > /tmp/essential.pkgs + sed -i /tmp/essential.pkgs -n -e 's/\([^ ]\+\).*yes$/\1/p' + cat /tmp/essential.pkgs >> $apt_altstate/requirements-deb.txt +} #apt_altstate=`mktemp -d --suffix="-apt-altstate"` apt_lists_dir="$apt_altstate/var/lib/apt/lists" @@ -54,12 +65,7 @@ if ! apt-get $apt_altstate_opts update; then exit 1 fi -if ! hash wget 2>/dev/null; then apt-get -y install wget; fi - -if ! wget -nv https://raw.githubusercontent.com/stackforge/fuel-main/${FUEL_BRANCH}/requirements-deb.txt -O $apt_altstate/requirements-deb.txt; then - echo "`basename $0`: failed to fetch requirements-deb.txt" - exit 1 -fi +requirements_add_essential_pkgs echo "Processing Fuel dependencies..." @@ -91,4 +97,4 @@ for dload in $NETBOOT_FILES; do echo NONE NONE dists/${UBUNTU_RELEASE}-updates/main/installer-${UBUNTU_ARCH}/current/images/${UBUNTU_NETBOOT_FLAVOR}/ubuntu-installer/${UBUNTU_ARCH}/${dload} >> $apt_altstate/netboot_md5.list done -exit 0 \ No newline at end of file +exit 0