openstack/devstack
Code Issues Proposed changes

Merge branch 'fix_cache' into lessapt

Browse Source 
Conflicts:
	tools/build_libvirt.sh
This commit is contained in:
Anthony Young 2011-11-03 16:31:20 -07:00
commit f7fe442ded
15 changed files with 90 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
exercise.sh
View File

@ -192,21 +192,23 @@ nova secgroup-delete $SECGROUP
# make sure that we can describe instances
euca-describe-instances
# Testing Swift
# =============
if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
# Testing Swift
# =============
# Check if we have to swift via keystone
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD stat
# Check if we have to swift via keystone
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD stat
# We start by creating a test container
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD post testcontainer
# We start by creating a test container
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD post testcontainer
# add some files into it.
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD upload testcontainer /etc/issue
# add some files into it.
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD upload testcontainer /etc/issue
# list them
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD list testcontainer
# list them
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD list testcontainer
# And we may want to delete them now that we have tested that
# everything works.
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD delete testcontainer
# And we may want to delete them now that we have tested that
# everything works.
swift --auth-version 2 -A http://${HOST_IP}:5000/v2.0 -U admin -K $ADMIN_PASSWORD delete testcontainer
fi

6
files/apts/nova
View File

@ -2,7 +2,7 @@ dnsmasq-base
kpartx
parted
arping # used for send_arp_for_ha option in nova-network
mysql-server
mysql-server # NOPRIME
python-mysqldb
kvm
gawk
@ -11,10 +11,10 @@ ebtables
sqlite3
sudo
kvm
libvirt-bin
libvirt-bin # NOPRIME
vlan
curl
rabbitmq-server
rabbitmq-server # NOPRIME
socat # used by ajaxterm
python-mox
python-paste

2
files/apts/swift
View File

@ -1,6 +1,6 @@
curl
gcc
memcached
memcached # NOPRIME
python-configobj
python-coverage
python-dev

1
files/swift/account-server.conf
View File

@ -4,6 +4,7 @@ mount_check = false
bind_port = %BIND_PORT%
user = %USER%
log_facility = LOG_LOCAL%LOG_FACILITY%
swift_dir = %SWIFT_CONFIG_LOCATION%
[pipeline:main]
pipeline = account-server

1
files/swift/container-server.conf
View File

@ -4,6 +4,7 @@ mount_check = false
bind_port = %BIND_PORT%
user = %USER%
log_facility = LOG_LOCAL%LOG_FACILITY%
swift_dir = %SWIFT_CONFIG_LOCATION%
[pipeline:main]
pipeline = container-server

1
files/swift/object-server.conf
View File

@ -4,6 +4,7 @@ mount_check = false
bind_port = %BIND_PORT%
user = %USER%
log_facility = LOG_LOCAL%LOG_FACILITY%
swift_dir = %SWIFT_CONFIG_LOCATION%
[pipeline:main]
pipeline = object-server

1
files/swift/proxy-server.conf
View File

@ -2,6 +2,7 @@
bind_port = 8080
user = %USER%
log_facility = LOG_LOCAL1
swift_dir = %SWIFT_CONFIG_LOCATION%
[pipeline:main]
pipeline = healthcheck cache %AUTH_SERVER% proxy-server

24
files/swift/rsyncd.conf
View File

@ -6,74 +6,74 @@ address = 127.0.0.1
[account6012]
max connections = 25
path = %SWIFT_LOCATION%/1/node/
path = %SWIFT_DATA_LOCATION%/1/node/
read only = false
lock file = /var/lock/account6012.lock
[account6022]
max connections = 25
path = %SWIFT_LOCATION%/2/node/
path = %SWIFT_DATA_LOCATION%/2/node/
read only = false
lock file = /var/lock/account6022.lock
[account6032]
max connections = 25
path = %SWIFT_LOCATION%/3/node/
path = %SWIFT_DATA_LOCATION%/3/node/
read only = false
lock file = /var/lock/account6032.lock
[account6042]
max connections = 25
path = %SWIFT_LOCATION%/4/node/
path = %SWIFT_DATA_LOCATION%/4/node/
read only = false
lock file = /var/lock/account6042.lock
[container6011]
max connections = 25
path = %SWIFT_LOCATION%/1/node/
path = %SWIFT_DATA_LOCATION%/1/node/
read only = false
lock file = /var/lock/container6011.lock
[container6021]
max connections = 25
path = %SWIFT_LOCATION%/2/node/
path = %SWIFT_DATA_LOCATION%/2/node/
read only = false
lock file = /var/lock/container6021.lock
[container6031]
max connections = 25
path = %SWIFT_LOCATION%/3/node/
path = %SWIFT_DATA_LOCATION%/3/node/
read only = false
lock file = /var/lock/container6031.lock
[container6041]
max connections = 25
path = %SWIFT_LOCATION%/4/node/
path = %SWIFT_DATA_LOCATION%/4/node/
read only = false
lock file = /var/lock/container6041.lock
[object6010]
max connections = 25
path = %SWIFT_LOCATION%/1/node/
path = %SWIFT_DATA_LOCATION%/1/node/
read only = false
lock file = /var/lock/object6010.lock
[object6020]
max connections = 25
path = %SWIFT_LOCATION%/2/node/
path = %SWIFT_DATA_LOCATION%/2/node/
read only = false
lock file = /var/lock/object6020.lock
[object6030]
max connections = 25
path = %SWIFT_LOCATION%/3/node/
path = %SWIFT_DATA_LOCATION%/3/node/
read only = false
lock file = /var/lock/object6030.lock
[object6040]
max connections = 25
path = %SWIFT_LOCATION%/4/node/
path = %SWIFT_DATA_LOCATION%/4/node/
read only = false
lock file = /var/lock/object6040.lock

2
files/swift/swift-remakerings
View File

@ -1,6 +1,6 @@
#!/bin/bash
cd /etc/swift
cd %SWIFT_CONFIG_LOCATION%
rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz

71
stack.sh
View File

@ -293,9 +293,13 @@ GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$HOST_IP:9292}
# TODO: add logging to different location.
# By default the location of swift drives and objects is located inside
# the swift source directory. SWIFT_LOCATION variable allow you to redefine
# the swift source directory. SWIFT_DATA_LOCATION variable allow you to redefine
# this.
SWIFT_LOCATION=${SWIFT_LOCATION:-${SWIFT_DIR}/data}
SWIFT_DATA_LOCATION=${SWIFT_DATA_LOCATION:-${SWIFT_DIR}/data}
# We are going to have the configuration files inside the source
# directory, change SWIFT_CONFIG_LOCATION if you want to adjust that.
SWIFT_CONFIG_LOCATION=${SWIFT_CONFIG_LOCATION:-${SWIFT_DIR}/config}
# devstack will create a loop-back disk formatted as XFS to store the
# swift data. By default the disk size is 1 gigabyte. The variable
@ -313,9 +317,13 @@ SWIFT_LOOPBACK_DISK_SIZE=${SWIFT_LOOPBACK_DISK_SIZE:-1000000}
# By default we define 9 for the partition count (which mean 512).
SWIFT_PARTITION_POWER_SIZE=${SWIFT_PARTITION_POWER_SIZE:-9}
# SWIFT_HASH is a random unique string for a swift cluster that can never change.
read_password SWIFT_HASH "ENTER A RANDOM SWIFT HASH."
# We only ask for Swift Hash if we have enabled swift service.
if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
# SWIFT_HASH is a random unique string for a swift cluster that
# can never change.
read_password SWIFT_HASH "ENTER A RANDOM SWIFT HASH."
fi
# Keystone
# --------
@ -640,47 +648,54 @@ if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
# changing the permissions so we can run it as our user.
USER_GROUP=$(id -g)
sudo mkdir -p ${SWIFT_LOCATION}/drives
sudo chown -R $USER:${USER_GROUP} ${SWIFT_LOCATION}/drives
sudo mkdir -p ${SWIFT_DATA_LOCATION}/drives
sudo chown -R $USER:${USER_GROUP} ${SWIFT_DATA_LOCATION}/drives
# We then create a loopback disk and format it to XFS.
if [[ ! -e ${SWIFT_LOCATION}/drives/images/swift.img ]];then
mkdir -p ${SWIFT_LOCATION}/drives/images
sudo touch ${SWIFT_LOCATION}/drives/images/swift.img
sudo chown $USER: ${SWIFT_LOCATION}/drives/images/swift.img
if [[ ! -e ${SWIFT_DATA_LOCATION}/drives/images/swift.img ]];then
mkdir -p ${SWIFT_DATA_LOCATION}/drives/images
sudo touch ${SWIFT_DATA_LOCATION}/drives/images/swift.img
sudo chown $USER: ${SWIFT_DATA_LOCATION}/drives/images/swift.img
dd if=/dev/zero of=${SWIFT_LOCATION}/drives/images/swift.img \
dd if=/dev/zero of=${SWIFT_DATA_LOCATION}/drives/images/swift.img \
bs=1024 count=0 seek=${SWIFT_LOOPBACK_DISK_SIZE}
mkfs.xfs -f -i size=1024 ${SWIFT_LOCATION}/drives/images/swift.img
mkfs.xfs -f -i size=1024 ${SWIFT_DATA_LOCATION}/drives/images/swift.img
fi
# After the drive being created we mount the disk with a few mount
# options to make it most efficient as possible for swift.
mkdir -p ${SWIFT_LOCATION}/drives/sdb1
if ! egrep -q ${SWIFT_LOCATION}/drives/sdb1 /proc/mounts;then
mkdir -p ${SWIFT_DATA_LOCATION}/drives/sdb1
if ! egrep -q ${SWIFT_DATA_LOCATION}/drives/sdb1 /proc/mounts;then
sudo mount -t xfs -o loop,noatime,nodiratime,nobarrier,logbufs=8 \
${SWIFT_LOCATION}/drives/images/swift.img ${SWIFT_LOCATION}/drives/sdb1
${SWIFT_DATA_LOCATION}/drives/images/swift.img ${SWIFT_DATA_LOCATION}/drives/sdb1
fi
# We then create link to that mounted location so swift would know
# where to go.
for x in {1..4}; do sudo ln -sf ${SWIFT_LOCATION}/drives/sdb1/$x ${SWIFT_LOCATION}/$x; done
for x in {1..4}; do sudo ln -sf ${SWIFT_DATA_LOCATION}/drives/sdb1/$x ${SWIFT_DATA_LOCATION}/$x; done
# We now have to emulate a few different servers into one we
# create all the directories needed for swift
tmpd=""
for d in ${SWIFT_LOCATION}/drives/sdb1/{1..4} /etc/swift /etc/swift/{object,container,account}-server \
${SWIFT_LOCATION}/{1..4}/node/sdb1 /var/run/swift ;do
for d in ${SWIFT_DATA_LOCATION}/drives/sdb1/{1..4} \
${SWIFT_CONFIG_LOCATION}/{object,container,account}-server \
${SWIFT_DATA_LOCATION}/{1..4}/node/sdb1 /var/run/swift ;do
[[ -d $d ]] && continue
sudo install -o ${USER} -g $USER_GROUP -d $d
done
sudo chown -R $USER: ${SWIFT_LOCATION}/{1..4}/node
# We do want to make sure this is all owned by our user.
sudo chown -R $USER: ${SWIFT_DATA_LOCATION}/{1..4}/node
sudo chown -R $USER: ${SWIFT_CONFIG_LOCATION}
# swift-init has a bug using /etc/swift until bug #885595 is fixed
# we have to create a link
sudo ln -s ${SWIFT_CONFIG_LOCATION} /etc/swift
# Swift use rsync to syncronize between all the different
# partitions (which make more sense when you have a multi-node
# setup) we configure it with our version of rsync.
sed -e "s/%GROUP%/${USER_GROUP}/;s/%USER%/$USER/;s,%SWIFT_LOCATION%,$SWIFT_LOCATION," $FILES/swift/rsyncd.conf | sudo tee /etc/rsyncd.conf
sed -e "s/%GROUP%/${USER_GROUP}/;s/%USER%/$USER/;s,%SWIFT_DATA_LOCATION%,$SWIFT_DATA_LOCATION," $FILES/swift/rsyncd.conf | sudo tee /etc/rsyncd.conf
sudo sed -i '/^RSYNC_ENABLE=false/ { s/false/true/ }' /etc/default/rsync
# By default Swift will be installed with the tempauth middleware
@ -699,10 +714,10 @@ if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
# We do the install of the proxy-server and swift configuration
# replacing a few directives to match our configuration.
sed "s/%USER%/$USER/;s/%SERVICE_TOKEN%/${SERVICE_TOKEN}/;s/%AUTH_SERVER%/${swift_auth_server}/" \
$FILES/swift/proxy-server.conf|sudo tee /etc/swift/proxy-server.conf
sed "s,%SWIFT_CONFIG_LOCATION%,${SWIFT_CONFIG_LOCATION},;s/%USER%/$USER/;s/%SERVICE_TOKEN%/${SERVICE_TOKEN}/;s/%AUTH_SERVER%/${swift_auth_server}/" \
$FILES/swift/proxy-server.conf|sudo tee ${SWIFT_CONFIG_LOCATION}/proxy-server.conf
sed -e "s/%SWIFT_HASH%/$SWIFT_HASH/" $FILES/swift/swift.conf > /etc/swift/swift.conf
sed -e "s/%SWIFT_HASH%/$SWIFT_HASH/" $FILES/swift/swift.conf > ${SWIFT_CONFIG_LOCATION}/swift.conf
# We need to generate a object/account/proxy configuration
# emulating 4 nodes on different ports we have a little function
@ -714,9 +729,9 @@ if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
local node_number
for node_number in {1..4};do
node_path=${SWIFT_LOCATION}/${node_number}
sed -e "s,%USER%,$USER,;s,%NODE_PATH%,${node_path},;s,%BIND_PORT%,${bind_port},;s,%LOG_FACILITY%,${log_facility}," \
$FILES/swift/${server_type}-server.conf > /etc/swift/${server_type}-server/${node_number}.conf
node_path=${SWIFT_DATA_LOCATION}/${node_number}
sed -e "s,%SWIFT_CONFIG_LOCATION%,${SWIFT_CONFIG_LOCATION},;s,%USER%,$USER,;s,%NODE_PATH%,${node_path},;s,%BIND_PORT%,${bind_port},;s,%LOG_FACILITY%,${log_facility}," \
$FILES/swift/${server_type}-server.conf > ${SWIFT_CONFIG_LOCATION}/${server_type}-server/${node_number}.conf
bind_port=$(( ${bind_port} + 10 ))
log_facility=$(( ${log_facility} + 1 ))
done
@ -732,7 +747,7 @@ if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
# - swift-startmain
# Restart your full cluster.
#
sed -e "s/%SWIFT_PARTITION_POWER_SIZE%/$SWIFT_PARTITION_POWER_SIZE/" $FILES/swift/swift-remakerings | \
sed -e "s,%SWIFT_CONFIG_LOCATION%,${SWIFT_CONFIG_LOCATION},;s/%SWIFT_PARTITION_POWER_SIZE%/$SWIFT_PARTITION_POWER_SIZE/" $FILES/swift/swift-remakerings | \
sudo tee /usr/local/bin/swift-remakerings
sudo install -m755 $FILES/swift/swift-startmain /usr/local/bin/
sudo chmod +x /usr/local/bin/swift-*

2
stackrc
View File

@ -4,7 +4,7 @@ NOVA_BRANCH=diablo
# storage service
SWIFT_REPO=https://github.com/openstack/swift.git
SWIFT_BRANCH=1.4.3
SWIFT_BRANCH=stable/diablo
# swift and keystone integration
SWIFT_KEYSTONE_REPO=https://github.com/cloudbuilders/swift-keystone2.git

5
tools/build_libvirt.sh
View File

@ -145,8 +145,9 @@ function git_clone {
# Make sure that base requirements are installed
cp /etc/resolv.conf $COPY_DIR/etc/resolv.conf
chroot $COPY_DIR apt-get update
chroot $COPY_DIR apt-get install -y --download-only `cat files/apts/* | cut -d\# -f1`
chroot $COPY_DIR apt-get install -y --force-yes `cat files/apts/general`
chroot $COPY_DIR apt-get install -y --download-only `cat files/apts/* | grep NOPRIME | cut -d\# -f1`
chroot $COPY_DIR apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1`
chroot $COPY_DIR pip install `cat files/pips/*`
# Clean out code repos if directed to do so
if [ "$CLEAN" = "1" ]; then

4
tools/build_lxc.sh
View File

@ -125,8 +125,8 @@ fi
# Make sure that base requirements are installed
chroot $CACHEDIR apt-get update
chroot $CACHEDIR apt-get install -y --force-yes `cat files/apts/* | cut -d\# -f1 | egrep -v "(rabbitmq|libvirt-bin|mysql-server|memcached)"`
chroot $CACHEDIR apt-get install -y --download-only rabbitmq-server libvirt-bin mysql-server
chroot $CACHEDIR apt-get install -y --download-only `cat files/apts/* | grep NOPRIME | cut -d\# -f1`
chroot $CACHEDIR apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1`
chroot $CACHEDIR pip install `cat files/pips/*`
# Clean out code repos if directed to do so

5
tools/build_nfs.sh
View File

@ -32,8 +32,9 @@ fi
# prime natty with as many apt/pips as we can
if [ ! -d $CHROOTCACHE/natty-dev ]; then
rsync -azH $CHROOTCACHE/natty-base/ $CHROOTCACHE/natty-dev/
chroot $CHROOTCACHE/natty-dev apt-get install -y `cat files/apts/* | cut -d\# -f1 | egrep -v "(rabbitmq|libvirt-bin|mysql-server)"`
chroot $CHROOTCACHE/natty-dev pip install `cat files/pips/*`
chroot $CHROOTCACHE apt-get install -y --download-only `cat files/apts/* | grep NOPRIME | cut -d\# -f1`
chroot $CHROOTCACHE apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1`
chroot $CHROOTCACHE pip install `cat files/pips/*`
# Create a stack user that is a member of the libvirtd group so that stack
# is able to interact with libvirt.

3
tools/build_ramdisk.sh
View File

@ -113,7 +113,8 @@ if [ ! -r $DEV_FILE ]; then
mount -t ext4 ${NBD}p1 $MNTDIR
cp -p /etc/resolv.conf $MNTDIR/etc/resolv.conf
chroot $MNTDIR apt-get install -y `cat files/apts/* | cut -d\# -f1 | egrep -v "(rabbitmq|libvirt-bin|mysql-server)"`
chroot $MNTDIR apt-get install -y --download-only `cat files/apts/* | grep NOPRIME | cut -d\# -f1`
chroot $MNTDIR apt-get install -y --force-yes `cat files/apts/* | grep -v NOPRIME | cut -d\# -f1`
chroot $MNTDIR pip install `cat files/pips/*`
# Create a stack user that is a member of the libvirtd group so that stack