diff --git a/contrib/nova.sh b/contrib/nova.sh new file mode 100755 index 000000000000..9bc36d6fbdbe --- /dev/null +++ b/contrib/nova.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +DIR=`pwd` +CMD=$1 +SOURCE_BRANCH=lp:nova +if [ -n "$2" ]; then + SOURCE_BRANCH=$2 +fi +DIRNAME=nova +NOVA_DIR=$DIR/$DIRNAME +if [ -n "$3" ]; then + NOVA_DIR=$DIR/$3 +fi + +if [ ! -n "$HOST_IP" ]; then + # NOTE(vish): This will just get the first ip in the list, so if you + # have more than one eth device set up, this will fail, and + # you should explicitly set HOST_IP in your environment + HOST_IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` +fi +TEST=0 +USE_MYSQL=0 +MYSQL_PASS=nova +USE_LDAP=0 +LIBVIRT_TYPE=qemu + +if [ "$USE_MYSQL" == 1 ]; then + SQL_CONN=mysql://root:$MYSQL_PASS@localhost/nova +else + SQL_CONN=sqlite:///$NOVA_DIR/nova.sqlite +fi + +if [ "$USE_LDAP" == 1 ]; then + AUTH=ldapdriver.LdapDriver +else + AUTH=dbdriver.DbDriver +fi + +mkdir -p /etc/nova +cat >/etc/nova/nova-manage.conf << NOVA_CONF_EOF +--verbose +--nodaemon +--dhcpbridge_flagfile=/etc/nova/nova-manage.conf +--FAKE_subdomain=ec2 +--cc_host=$HOST_IP +--routing_source_ip=$HOST_IP +--sql_connection=$SQL_CONN +--auth_driver=nova.auth.$AUTH +--libvirt_type=$LIBVIRT_TYPE +NOVA_CONF_EOF + +if [ "$CMD" == "branch" ]; then + sudo apt-get install -y bzr + rm -rf $NOVA_DIR + bzr branch $SOURCE_BRANCH $NOVA_DIR + cd $NOVA_DIR + mkdir -p $NOVA_DIR/instances + mkdir -p $NOVA_DIR/networks +fi + +# You should only have to run this once +if [ "$CMD" == "install" ]; then + sudo apt-get install -y python-software-properties + sudo add-apt-repository ppa:nova-core/ppa + sudo apt-get update + sudo apt-get install -y dnsmasq open-iscsi kpartx kvm gawk iptables ebtables + sudo apt-get install -y user-mode-linux kvm libvirt-bin + sudo apt-get install -y screen iscsitarget euca2ools vlan curl rabbitmq-server + sudo modprobe kvm + sudo /etc/init.d/libvirt-bin restart + sudo apt-get install -y python-twisted python-sqlalchemy python-mox python-greenlet python-carrot + sudo apt-get install -y python-daemon python-eventlet python-gflags python-tornado python-ipy + sudo apt-get install -y python-libvirt python-libxml2 python-routes + if [ "$USE_MYSQL" == 1 ]; then + cat < + %(name)s + + hvm + %(basepath)s/kernel + %(basepath)s/ramdisk + root=/dev/vda1 console=ttyS0 + + + + + %(memory_kb)s + %(vcpus)s + + + + + + + + + + + + + + + + + diff --git a/contrib/puppet/files/production/my.cnf b/contrib/puppet/files/production/my.cnf new file mode 100644 index 000000000000..8777bc4806bd --- /dev/null +++ b/contrib/puppet/files/production/my.cnf @@ -0,0 +1,137 @@ +# +# The MySQL database server configuration file. +# +# You can copy this to one of: +# - "/etc/mysql/my.cnf" to set global options, +# - "~/.my.cnf" to set user-specific options. +# +# One can use all long options that the program supports. +# Run program with --help to get a list of available options and with +# --print-defaults to see which it would actually understand and use. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +# This will be passed to all mysql clients +# It has been reported that passwords should be enclosed with ticks/quotes +# escpecially if they contain "#" chars... +# Remember to edit /etc/mysql/debian.cnf when changing the socket location. +[client] +port = 3306 +socket = /var/run/mysqld/mysqld.sock + +# Here is entries for some specific programs +# The following values assume you have at least 32M ram + +# This was formally known as [safe_mysqld]. Both versions are currently parsed. +[mysqld_safe] +socket = /var/run/mysqld/mysqld.sock +nice = 0 + +[mysqld] +# +# * Basic Settings +# + +# +# * IMPORTANT +# If you make changes to these settings and your system uses apparmor, you may +# also need to also adjust /etc/apparmor.d/usr.sbin.mysqld. +# + +user = mysql +socket = /var/run/mysqld/mysqld.sock +port = 3306 +basedir = /usr +datadir = /var/lib/mysql +tmpdir = /tmp +skip-external-locking +# +# Instead of skip-networking the default is now to listen only on +# localhost which is more compatible and is not less secure. +# bind-address = 127.0.0.1 +# +# * Fine Tuning +# +innodb_buffer_pool_size = 12G +#innodb_log_file_size = 256M +innodb_log_buffer_size=4M +innodb_flush_log_at_trx_commit=2 +innodb_thread_concurrency=8 +innodb_flush_method=O_DIRECT +key_buffer = 128M +max_allowed_packet = 256M +thread_stack = 8196K +thread_cache_size = 32 +# This replaces the startup script and checks MyISAM tables if needed +# the first time they are touched +myisam-recover = BACKUP +max_connections = 1000 +table_cache = 1024 +#thread_concurrency = 10 +# +# * Query Cache Configuration +# +query_cache_limit = 32M +query_cache_size = 256M +# +# * Logging and Replication +# +# Both location gets rotated by the cronjob. +# Be aware that this log type is a performance killer. +# As of 5.1 you can enable the log at runtime! +#general_log_file = /var/log/mysql/mysql.log +#general_log = 1 + +log_error = /var/log/mysql/error.log + +# Here you can see queries with especially long duration +log_slow_queries = /var/log/mysql/mysql-slow.log +long_query_time = 2 +#log-queries-not-using-indexes +# +# The following can be used as easy to replay backup logs or for replication. +# note: if you are setting up a replication slave, see README.Debian about +# other settings you may need to change. +server-id = 1 +log_bin = /var/log/mysql/mysql-bin.log +expire_logs_days = 10 +max_binlog_size = 50M +#binlog_do_db = include_database_name +#binlog_ignore_db = include_database_name +# +# * InnoDB +# +sync_binlog=1 +# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. +# Read the manual for more InnoDB related options. There are many! +# +# * Security Features +# +# Read the manual, too, if you want chroot! +# chroot = /var/lib/mysql/ +# +# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". +# +# ssl-ca=/etc/mysql/cacert.pem +# ssl-cert=/etc/mysql/server-cert.pem +# ssl-key=/etc/mysql/server-key.pem + + + +[mysqldump] +quick +quote-names +max_allowed_packet = 256M + +[mysql] +#no-auto-rehash # faster start of mysql but no tab completition + +[isamchk] +key_buffer = 128M + +# +# * IMPORTANT: Additional settings that can override those from this file! +# The files must end with '.cnf', otherwise they'll be ignored. +# +!includedir /etc/mysql/conf.d/ diff --git a/contrib/puppet/files/production/nova-iptables b/contrib/puppet/files/production/nova-iptables new file mode 100755 index 000000000000..b7b52df87753 --- /dev/null +++ b/contrib/puppet/files/production/nova-iptables @@ -0,0 +1,185 @@ +#! /bin/sh + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(vish): This script sets up some reasonable defaults for iptables and +# creates nova-specific chains. If you use this script you should +# run nova-network and nova-compute with --use_nova_chains=True + + +# NOTE(vish): If you run public nova-api on a different port, make sure to +# change the port here + +if [ -f /etc/default/nova-iptables ] ; then + . /etc/default/nova-iptables +fi + +API_PORT=${API_PORT:-"8773"} + +if [ ! -n "$IP" ]; then + # NOTE(vish): IP address is what address the services ALLOW on. + # This will just get the first ip in the list, so if you + # have more than one eth device set up, this will fail, and + # you should explicitly pass in the ip of the instance + IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` +fi + +if [ ! -n "$PRIVATE_RANGE" ]; then + #NOTE(vish): PRIVATE_RANGE: range is ALLOW to access DHCP + PRIVATE_RANGE="192.168.0.0/12" +fi + +if [ ! -n "$MGMT_IP" ]; then + # NOTE(vish): Management IP is the ip over which to allow ssh traffic. It + # will also allow traffic to nova-api + MGMT_IP="$IP" +fi + +if [ ! -n "$DMZ_IP" ]; then + # NOTE(vish): DMZ IP is the ip over which to allow api & objectstore access + DMZ_IP="$IP" +fi + +clear_nova_iptables() { + iptables -P INPUT ACCEPT + iptables -P FORWARD ACCEPT + iptables -P OUTPUT ACCEPT + iptables -F + iptables -t nat -F + iptables -F services + iptables -X services + # HACK: re-adding fail2ban rules :( + iptables -N fail2ban-ssh + iptables -A INPUT -p tcp -m multiport --dports 22 -j fail2ban-ssh + iptables -A fail2ban-ssh -j RETURN +} + +load_nova_iptables() { + + iptables -P INPUT DROP + iptables -A INPUT -m state --state INVALID -j DROP + iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + # NOTE(ja): allow localhost for everything + iptables -A INPUT -d 127.0.0.1/32 -j ACCEPT + # NOTE(ja): 22 only allowed MGMT_IP before, but we widened it to any + # address, since ssh should be listening only on internal + # before we re-add this rule we will need to add + # flexibility for RSYNC between omega/stingray + iptables -A INPUT -m tcp -p tcp --dport 22 -j ACCEPT + iptables -A INPUT -m udp -p udp --dport 123 -j ACCEPT + iptables -A INPUT -p icmp -j ACCEPT + iptables -N services + iptables -A INPUT -j services + iptables -A INPUT -p tcp -j REJECT --reject-with tcp-reset + iptables -A INPUT -j REJECT --reject-with icmp-port-unreachable + + iptables -P FORWARD DROP + iptables -A FORWARD -m state --state INVALID -j DROP + iptables -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT + iptables -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu + + # NOTE(vish): DROP on output is too restrictive for now. We need to add + # in a bunch of more specific output rules to use it. + # iptables -P OUTPUT DROP + iptables -A OUTPUT -m state --state INVALID -j DROP + iptables -A OUTPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + + if [ -n "$GANGLIA" ] || [ -n "$ALL" ]; then + iptables -A services -m tcp -p tcp -d $IP --dport 8649 -j ACCEPT + iptables -A services -m udp -p udp -d $IP --dport 8649 -j ACCEPT + fi + + # if [ -n "$WEB" ] || [ -n "$ALL" ]; then + # # NOTE(vish): This opens up ports for web access, allowing web-based + # # dashboards to work. + # iptables -A services -m tcp -p tcp -d $IP --dport 80 -j ACCEPT + # iptables -A services -m tcp -p tcp -d $IP --dport 443 -j ACCEPT + # fi + + if [ -n "$OBJECTSTORE" ] || [ -n "$ALL" ]; then + # infrastructure + iptables -A services -m tcp -p tcp -d $IP --dport 3333 -j ACCEPT + # clients + iptables -A services -m tcp -p tcp -d $DMZ_IP --dport 3333 -j ACCEPT + fi + + if [ -n "$API" ] || [ -n "$ALL" ]; then + iptables -A services -m tcp -p tcp -d $IP --dport $API_PORT -j ACCEPT + if [ "$IP" != "$DMZ_IP" ]; then + iptables -A services -m tcp -p tcp -d $DMZ_IP --dport $API_PORT -j ACCEPT + fi + if [ "$IP" != "$MGMT_IP" ] && [ "$DMZ_IP" != "$MGMT_IP" ]; then + iptables -A services -m tcp -p tcp -d $MGMT_IP --dport $API_PORT -j ACCEPT + fi + fi + + if [ -n "$REDIS" ] || [ -n "$ALL" ]; then + iptables -A services -m tcp -p tcp -d $IP --dport 6379 -j ACCEPT + fi + + if [ -n "$MYSQL" ] || [ -n "$ALL" ]; then + iptables -A services -m tcp -p tcp -d $IP --dport 3306 -j ACCEPT + fi + + if [ -n "$RABBITMQ" ] || [ -n "$ALL" ]; then + iptables -A services -m tcp -p tcp -d $IP --dport 4369 -j ACCEPT + iptables -A services -m tcp -p tcp -d $IP --dport 5672 -j ACCEPT + iptables -A services -m tcp -p tcp -d $IP --dport 53284 -j ACCEPT + fi + + if [ -n "$DNSMASQ" ] || [ -n "$ALL" ]; then + # NOTE(vish): this could theoretically be setup per network + # for each host, but it seems like overkill + iptables -A services -m tcp -p tcp -s $PRIVATE_RANGE --dport 53 -j ACCEPT + iptables -A services -m udp -p udp -s $PRIVATE_RANGE --dport 53 -j ACCEPT + iptables -A services -m udp -p udp --dport 67 -j ACCEPT + fi + + if [ -n "$LDAP" ] || [ -n "$ALL" ]; then + iptables -A services -m tcp -p tcp -d $IP --dport 389 -j ACCEPT + fi + + if [ -n "$ISCSI" ] || [ -n "$ALL" ]; then + iptables -A services -m tcp -p tcp -d $IP --dport 3260 -j ACCEPT + iptables -A services -m tcp -p tcp -d 127.0.0.0/16 --dport 3260 -j ACCEPT + fi +} + + +case "$1" in + start) + echo "Starting nova-iptables: " + load_nova_iptables + ;; + stop) + echo "Clearing nova-iptables: " + clear_nova_iptables + ;; + restart) + echo "Restarting nova-iptables: " + clear_nova_iptables + load_nova_iptables + ;; + *) + echo "Usage: $NAME {start|stop|restart}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/contrib/puppet/files/production/nova-iscsi-dev.sh b/contrib/puppet/files/production/nova-iscsi-dev.sh new file mode 100644 index 000000000000..8eda10d2e331 --- /dev/null +++ b/contrib/puppet/files/production/nova-iscsi-dev.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# FILE: /etc/udev/scripts/iscsidev.sh + +BUS=${1} +HOST=${BUS%%:*} + +[ -e /sys/class/iscsi_host ] || exit 1 + +file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" + +target_name=$(cat ${file}) + +# This is not an open-scsi drive +if [ -z "${target_name}" ]; then + exit 1 +fi + +echo "${target_name##*:}" diff --git a/contrib/puppet/files/production/setup_data.sh b/contrib/puppet/files/production/setup_data.sh new file mode 100755 index 000000000000..1fbbac41c4d9 --- /dev/null +++ b/contrib/puppet/files/production/setup_data.sh @@ -0,0 +1,6 @@ +#!/bin/bash +/root/slap.sh +mysql -e "DROP DATABASE nova" +mysql -e "CREATE DATABASE nova" +mysql -e "GRANT ALL on nova.* to nova@'%' identified by 'TODO:CHANGEME:CMON'" +touch /root/installed diff --git a/contrib/puppet/files/production/slap.sh b/contrib/puppet/files/production/slap.sh new file mode 100755 index 000000000000..f8ea16949588 --- /dev/null +++ b/contrib/puppet/files/production/slap.sh @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# LDAP INSTALL SCRIPT - SHOULD BE IDEMPOTENT, but it SCRUBS all USERS + +apt-get install -y slapd ldap-utils python-ldap + +cat >/etc/ldap/schema/openssh-lpk_openldap.schema < +# +# Based on the proposal of : Mark Ruijter +# + + +# octetString SYNTAX +attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' + DESC 'MANDATORY: OpenSSH Public key' + EQUALITY octetStringMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) + +# printableString SYNTAX yes|no +objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY + DESC 'MANDATORY: OpenSSH LPK objectclass' + MAY ( sshPublicKey $ uid ) + ) +LPK_SCHEMA_EOF + +cat >/etc/ldap/schema/nova.schema < +# +# + +# using internet experimental oid arc as per BP64 3.1 +objectidentifier novaSchema 1.3.6.1.3.1.666.666 +objectidentifier novaAttrs novaSchema:3 +objectidentifier novaOCs novaSchema:4 + +attributetype ( + novaAttrs:1 + NAME 'accessKey' + DESC 'Key for accessing data' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:2 + NAME 'secretKey' + DESC 'Secret key' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:3 + NAME 'keyFingerprint' + DESC 'Fingerprint of private key' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:4 + NAME 'isAdmin' + DESC 'Is user an administrator?' + EQUALITY booleanMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:5 + NAME 'projectManager' + DESC 'Project Managers of a project' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + ) + +objectClass ( + novaOCs:1 + NAME 'novaUser' + DESC 'access and secret keys' + AUXILIARY + MUST ( uid ) + MAY ( accessKey $ secretKey $ isAdmin ) + ) + +objectClass ( + novaOCs:2 + NAME 'novaKeyPair' + DESC 'Key pair for User' + SUP top + STRUCTURAL + MUST ( cn $ sshPublicKey $ keyFingerprint ) + ) + +objectClass ( + novaOCs:3 + NAME 'novaProject' + DESC 'Container for project' + SUP groupOfNames + STRUCTURAL + MUST ( cn $ projectManager ) + ) + +NOVA_SCHEMA_EOF + +mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig +cat >/etc/ldap/slapd.conf </etc/ldap/ldap.conf </etc/ldap/base.ldif < "/usr/bin/apt-get update" } diff --git a/contrib/puppet/manifests/classes/issue.pp b/contrib/puppet/manifests/classes/issue.pp new file mode 100644 index 000000000000..8bb37ee3f826 --- /dev/null +++ b/contrib/puppet/manifests/classes/issue.pp @@ -0,0 +1,14 @@ +class issue { + file { "/etc/issue": + owner => "root", + group => "root", + mode => 444, + source => "puppet://${puppet_server}/files/etc/issue", + } + file { "/etc/issue.net": + owner => "root", + group => "root", + mode => 444, + source => "puppet://${puppet_server}/files/etc/issue", + } +} diff --git a/contrib/puppet/manifests/classes/kern_module.pp b/contrib/puppet/manifests/classes/kern_module.pp new file mode 100644 index 000000000000..00ec0636c8c7 --- /dev/null +++ b/contrib/puppet/manifests/classes/kern_module.pp @@ -0,0 +1,34 @@ +# via http://projects.puppetlabs.com/projects/puppet/wiki/Kernel_Modules_Patterns + +define kern_module ($ensure) { + $modulesfile = $operatingsystem ? { ubuntu => "/etc/modules", redhat => "/etc/rc.modules" } + case $operatingsystem { + redhat: { file { "/etc/rc.modules": ensure => file, mode => 755 } } + } + case $ensure { + present: { + exec { "insert_module_${name}": + command => $operatingsystem ? { + ubuntu => "/bin/echo '${name}' >> '${modulesfile}'", + redhat => "/bin/echo '/sbin/modprobe ${name}' >> '${modulesfile}' " + }, + unless => "/bin/grep -qFx '${name}' '${modulesfile}'" + } + exec { "/sbin/modprobe ${name}": unless => "/bin/grep -q '^${name} ' '/proc/modules'" } + } + absent: { + exec { "/sbin/modprobe -r ${name}": onlyif => "/bin/grep -q '^${name} ' '/proc/modules'" } + exec { "remove_module_${name}": + command => $operatingsystem ? { + ubuntu => "/usr/bin/perl -ni -e 'print unless /^\\Q${name}\\E\$/' '${modulesfile}'", + redhat => "/usr/bin/perl -ni -e 'print unless /^\\Q/sbin/modprobe ${name}\\E\$/' '${modulesfile}'" + }, + onlyif => $operatingsystem ? { + ubuntu => "/bin/grep -qFx '${name}' '${modulesfile}'", + redhat => "/bin/grep -q '^/sbin/modprobe ${name}' '${modulesfile}'" + } + } + } + default: { err ( "unknown ensure value ${ensure}" ) } + } +} diff --git a/contrib/puppet/manifests/classes/loopback.pp b/contrib/puppet/manifests/classes/loopback.pp new file mode 100644 index 000000000000..e0fa9d541868 --- /dev/null +++ b/contrib/puppet/manifests/classes/loopback.pp @@ -0,0 +1,6 @@ +define loopback($num) { + exec { "mknod -m 0660 /dev/loop${num} b 7 ${num}; chown root:disk /dev/loop${num}": + creates => "/dev/loop${num}", + path => ["/usr/bin", "/usr/sbin", "/bin"] + } +} diff --git a/contrib/puppet/manifests/classes/lvm.pp b/contrib/puppet/manifests/classes/lvm.pp new file mode 100644 index 000000000000..5a407abcba95 --- /dev/null +++ b/contrib/puppet/manifests/classes/lvm.pp @@ -0,0 +1,8 @@ +class lvm { + file { "/etc/lvm/lvm.conf": + owner => "root", + group => "root", + mode => 444, + source => "puppet://${puppet_server}/files/etc/lvm.conf", + } +} diff --git a/contrib/puppet/manifests/classes/lvmconf.pp b/contrib/puppet/manifests/classes/lvmconf.pp new file mode 100644 index 000000000000..4aa7ddfdc314 --- /dev/null +++ b/contrib/puppet/manifests/classes/lvmconf.pp @@ -0,0 +1,8 @@ +class lvmconf { + file { "/etc/lvm/lvm.conf": + owner => "root", group => "root", mode => 644, + source => "puppet://${puppet_server}/files/etc/lvm/lvm.conf", + ensure => present + } +} + diff --git a/contrib/puppet/manifests/classes/nova.pp b/contrib/puppet/manifests/classes/nova.pp new file mode 100644 index 000000000000..e942860f41d7 --- /dev/null +++ b/contrib/puppet/manifests/classes/nova.pp @@ -0,0 +1,464 @@ +import "kern_module" +import "apt" +import "loopback" + +#$head_node_ip = "undef" +#$rabbit_ip = "undef" +#$vpn_ip = "undef" +#$public_interface = "undef" +#$vlan_start = "5000" +#$vlan_end = "6000" +#$private_range = "10.0.0.0/16" +#$public_range = "192.168.177.0/24" + +define nova_iptables($services, $ip="", $private_range="", $mgmt_ip="", $dmz_ip="") { + file { "/etc/init.d/nova-iptables": + owner => "root", mode => 755, + source => "puppet://${puppet_server}/files/production/nova-iptables", + } + + file { "/etc/default/nova-iptables": + owner => "root", mode => 644, + content => template("nova-iptables.erb") + } +} + +define nova_conf_pointer($name) { + file { "/etc/nova/nova-${name}.conf": + owner => "nova", mode => 400, + content => "--flagfile=/etc/nova/nova.conf" + } +} + +class novaconf { + file { "/etc/nova/nova.conf": + owner => "nova", mode => 400, + content => template("production/nova-common.conf.erb", "production/nova-${cluster_name}.conf.erb") + } + nova_conf_pointer{'manage': name => 'manage'} +} + +class novadata { + package { "rabbitmq-server": ensure => present } + + file { "/etc/rabbitmq/rabbitmq.conf": + owner => "root", mode => 644, + content => "NODENAME=rabbit@localhost", + } + + service { "rabbitmq-server": + ensure => running, + enable => true, + hasstatus => true, + require => [ + File["/etc/rabbitmq/rabbitmq.conf"], + Package["rabbitmq-server"] + ] + } + + package { "mysql-server": ensure => present } + + file { "/etc/mysql/my.cnf": + owner => "root", mode => 644, + source => "puppet://${puppet_server}/files/production/my.cnf", + } + + service { "mysql": + ensure => running, + enable => true, + hasstatus => true, + require => [ + File["/etc/mysql/my.cnf"], + Package["mysql-server"] + ] + } + + file { "/root/slap.sh": + owner => "root", mode => 755, + source => "puppet://${puppet_server}/files/production/slap.sh", + } + + file { "/root/setup_data.sh": + owner => "root", mode => 755, + source => "puppet://${puppet_server}/files/production/setup_data.sh", + } + + # setup compute data + exec { "setup_data": + command => "/root/setup_data.sh", + path => "/usr/bin:/bin", + unless => "test -f /root/installed", + require => [ + Service["mysql"], + File["/root/slap.sh"], + File["/root/setup_data.sh"] + ] + } +} + +define nscheduler($version) { + package { "nova-scheduler": ensure => $version, require => Exec["update-apt"] } + nova_conf_pointer{'scheduler': name => 'scheduler'} + exec { "update-rc.d -f nova-scheduler remove; update-rc.d nova-scheduler defaults 50": + path => "/usr/bin:/usr/sbin:/bin", + onlyif => "test -f /etc/init.d/nova-scheduler", + unless => "test -f /etc/rc2.d/S50nova-scheduler" + } + service { "nova-scheduler": + ensure => running, + hasstatus => true, + subscribe => [ + Package["nova-scheduler"], + File["/etc/nova/nova.conf"], + File["/etc/nova/nova-scheduler.conf"] + ] + } + +} + +define napi($version, $api_servers, $api_base_port) { + file { "/etc/boto.cfg": + owner => "root", mode => 644, + source => "puppet://${puppet_server}/files/production/boto.cfg", + } + + file { "/var/lib/nova/CA/genvpn.sh": + owner => "nova", mode => 755, + source => "puppet://${puppet_server}/files/production/genvpn.sh", + } + + package { "python-greenlet": ensure => present } + package { "nova-api": ensure => $version, require => [Exec["update-apt"], Package["python-greenlet"]] } + nova_conf_pointer{'api': name => 'api'} + + exec { "update-rc.d -f nova-api remove; update-rc.d nova-api defaults 50": + path => "/usr/bin:/usr/sbin:/bin", + onlyif => "test -f /etc/init.d/nova-api", + unless => "test -f /etc/rc2.d/S50nova-api" + } + + service { "nova-netsync": + start => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock start", + stop => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock stop", + ensure => running, + hasstatus => false, + pattern => "nova-netsync", + require => Service["nova-api"], + subscribe => File["/etc/nova/nova.conf"] + } + service { "nova-api": + start => "monit start all -g nova_api", + stop => "monit stop all -g nova_api", + restart => "monit restart all -g nova_api", + # ensure => running, + # hasstatus => true, + require => Service["monit"], + subscribe => [ + Package["nova-objectstore"], + File["/etc/boto.cfg"], + File["/etc/nova/nova.conf"], + File["/etc/nova/nova-objectstore.conf"] + ] + } + + # the haproxy & monit's template use $api_servers and $api_base_port + + package { "haproxy": ensure => present } + file { "/etc/default/haproxy": + owner => "root", mode => 644, + content => "ENABLED=1", + require => Package['haproxy'] + } + file { "/etc/haproxy/haproxy.cfg": + owner => "root", mode => 644, + content => template("/srv/cloud/puppet/templates/haproxy.cfg.erb"), + require => Package['haproxy'] + } + service { "haproxy": + ensure => true, + enable => true, + hasstatus => true, + subscribe => [ + Package["haproxy"], + File["/etc/default/haproxy"], + File["/etc/haproxy/haproxy.cfg"], + ] + } + + package { "socat": ensure => present } + + file { "/usr/local/bin/gmetric_haproxy.sh": + owner => "root", mode => 755, + source => "puppet://${puppet_server}/files/production/ganglia/gmetric_scripts/gmetric_haproxy.sh", + } + + cron { "gmetric_haproxy": + command => "/usr/local/bin/gmetric_haproxy.sh", + user => root, + minute => "*/3", + } + + package { "monit": ensure => present } + + file { "/etc/default/monit": + owner => "root", mode => 644, + content => "startup=1", + require => Package['monit'] + } + file { "/etc/monit/monitrc": + owner => "root", mode => 600, + content => template("/srv/cloud/puppet/templates/monitrc-nova-api.erb"), + require => Package['monit'] + } + service { "monit": + ensure => true, + pattern => "sbin/monit", + subscribe => [ + Package["monit"], + File["/etc/default/monit"], + File["/etc/monit/monitrc"], + ] + } + +} + + +define nnetwork($version) { + # kill the default network added by the package + exec { "kill-libvirt-default-net": + command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", + path => "/usr/bin:/bin", + onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" + } + + # EVIL HACK: custom binary because dnsmasq 2.52 segfaulted accessing dereferenced object + file { "/usr/sbin/dnsmasq": + owner => "root", group => "root", + source => "puppet://${puppet_server}/files/production/dnsmasq", + } + + package { "nova-network": ensure => $version, require => Exec["update-apt"] } + nova_conf_pointer{'dhcpbridge': name => 'dhcpbridge'} + nova_conf_pointer{'network': name => "network" } + + exec { "update-rc.d -f nova-network remove; update-rc.d nova-network defaults 50": + path => "/usr/bin:/usr/sbin:/bin", + onlyif => "test -f /etc/init.d/nova-network", + unless => "test -f /etc/rc2.d/S50nova-network" + } + service { "nova-network": + ensure => running, + hasstatus => true, + subscribe => [ + Package["nova-network"], + File["/etc/nova/nova.conf"], + File["/etc/nova/nova-network.conf"] + ] + } +} + +define nobjectstore($version) { + package { "nova-objectstore": ensure => $version, require => Exec["update-apt"] } + nova_conf_pointer{'objectstore': name => 'objectstore'} + exec { "update-rc.d -f nova-objectstore remove; update-rc.d nova-objectstore defaults 50": + path => "/usr/bin:/usr/sbin:/bin", + onlyif => "test -f /etc/init.d/nova-objectstore", + unless => "test -f /etc/rc2.d/S50nova-objectstore" + } + service { "nova-objectstore": + ensure => running, + hasstatus => true, + subscribe => [ + Package["nova-objectstore"], + File["/etc/nova/nova.conf"], + File["/etc/nova/nova-objectstore.conf"] + ] + } +} + +define ncompute($version) { + include ganglia-python + include ganglia-compute + + # kill the default network added by the package + exec { "kill-libvirt-default-net": + command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", + path => "/usr/bin:/bin", + onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" + } + + + # LIBVIRT has to be restarted when ebtables / gawk is installed + service { "libvirt-bin": + ensure => running, + pattern => "sbin/libvirtd", + subscribe => [ + Package["ebtables"], + Kern_module["kvm_intel"] + ], + require => [ + Package["libvirt-bin"], + Package["ebtables"], + Package["gawk"], + Kern_module["kvm_intel"], + File["/dev/kvm"] + ] + } + + package { "libvirt-bin": ensure => "0.8.3-1ubuntu14~ppalucid2" } + package { "ebtables": ensure => present } + package { "gawk": ensure => present } + + # ensure proper permissions on /dev/kvm + file { "/dev/kvm": + owner => "root", + group => "kvm", + mode => 660 + } + + # require hardware virt + kern_module { "kvm_intel": + ensure => present, + } + + # increase loopback devices + file { "/etc/modprobe.d/loop.conf": + owner => "root", mode => 644, + content => "options loop max_loop=40" + } + + nova_conf_pointer{'compute': name => 'compute'} + + loopback{loop0: num => 0} + loopback{loop1: num => 1} + loopback{loop2: num => 2} + loopback{loop3: num => 3} + loopback{loop4: num => 4} + loopback{loop5: num => 5} + loopback{loop6: num => 6} + loopback{loop7: num => 7} + loopback{loop8: num => 8} + loopback{loop9: num => 9} + loopback{loop10: num => 10} + loopback{loop11: num => 11} + loopback{loop12: num => 12} + loopback{loop13: num => 13} + loopback{loop14: num => 14} + loopback{loop15: num => 15} + loopback{loop16: num => 16} + loopback{loop17: num => 17} + loopback{loop18: num => 18} + loopback{loop19: num => 19} + loopback{loop20: num => 20} + loopback{loop21: num => 21} + loopback{loop22: num => 22} + loopback{loop23: num => 23} + loopback{loop24: num => 24} + loopback{loop25: num => 25} + loopback{loop26: num => 26} + loopback{loop27: num => 27} + loopback{loop28: num => 28} + loopback{loop29: num => 29} + loopback{loop30: num => 30} + loopback{loop31: num => 31} + loopback{loop32: num => 32} + loopback{loop33: num => 33} + loopback{loop34: num => 34} + loopback{loop35: num => 35} + loopback{loop36: num => 36} + loopback{loop37: num => 37} + loopback{loop38: num => 38} + loopback{loop39: num => 39} + + package { "python-libvirt": ensure => "0.8.3-1ubuntu14~ppalucid2" } + + package { "nova-compute": + ensure => "$version", + require => Package["python-libvirt"] + } + + #file { "/usr/share/nova/libvirt.qemu.xml.template": + # owner => "nova", mode => 400, + # source => "puppet://${puppet_server}/files/production/libvirt.qemu.xml.template", + #} + + # fix runlevels: using enable => true adds it as 20, which is too early + exec { "update-rc.d -f nova-compute remove": + path => "/usr/bin:/usr/sbin:/bin", + onlyif => "test -f /etc/rc2.d/S??nova-compute" + } + service { "nova-compute": + ensure => running, + hasstatus => true, + subscribe => [ + Package["nova-compute"], + File["/etc/nova/nova.conf"], + File["/etc/nova/nova-compute.conf"], + #File["/usr/share/nova/libvirt.qemu.xml.template"], + Service["libvirt-bin"], + Kern_module["kvm_intel"] + ] + } +} + +define nvolume($version) { + + package { "nova-volume": ensure => $version, require => Exec["update-apt"] } + + nova_conf_pointer{'volume': name => 'volume'} + + # fix runlevels: using enable => true adds it as 20, which is too early + exec { "update-rc.d -f nova-volume remove": + path => "/usr/bin:/usr/sbin:/bin", + onlyif => "test -f /etc/rc2.d/S??nova-volume" + } + + file { "/etc/default/iscsitarget": + owner => "root", mode => 644, + content => "ISCSITARGET_ENABLE=true" + } + + package { "iscsitarget": ensure => present } + + file { "/dev/iscsi": ensure => directory } # FIXME(vish): owner / mode? + file { "/usr/sbin/nova-iscsi-dev.sh": + owner => "root", mode => 755, + source => "puppet://${puppet_server}/files/production/nova-iscsi-dev.sh" + } + file { "/etc/udev/rules.d/55-openiscsi.rules": + owner => "root", mode => 644, + content => 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/usr/sbin/nova-iscsi-dev.sh %b",SYMLINK+="iscsi/%c%n"' + } + + service { "iscsitarget": + ensure => running, + enable => true, + hasstatus => true, + require => [ + File["/etc/default/iscsitarget"], + Package["iscsitarget"] + ] + } + + service { "nova-volume": + ensure => running, + hasstatus => true, + subscribe => [ + Package["nova-volume"], + File["/etc/nova/nova.conf"], + File["/etc/nova/nova-volume.conf"] + ] + } +} + +class novaspool { + # This isn't in release yet + #cron { logspool: + # command => "/usr/bin/nova-logspool /var/log/nova.log /var/lib/nova/spool", + # user => "nova" + #} + #cron { spoolsentry: + # command => "/usr/bin/nova-spoolsentry ${sentry_url} ${sentry_key} /var/lib/nova/spool", + # user => "nova" + #} +} diff --git a/contrib/puppet/manifests/classes/swift.pp b/contrib/puppet/manifests/classes/swift.pp new file mode 100644 index 000000000000..64ffb6fa3877 --- /dev/null +++ b/contrib/puppet/manifests/classes/swift.pp @@ -0,0 +1,7 @@ +class swift { + package { "memcached": ensure => present } + service { "memcached": require => Package['memcached'] } + + package { "swift-proxy": ensure => present } +} + diff --git a/contrib/puppet/manifests/site.pp b/contrib/puppet/manifests/site.pp new file mode 100644 index 000000000000..ca07a34ad41b --- /dev/null +++ b/contrib/puppet/manifests/site.pp @@ -0,0 +1,120 @@ +# site.pp + +import "templates" +import "classes/*" + +node novabase inherits default { +# $puppet_server = "192.168.0.10" + $cluster_name = "openstack001" + $ganglia_udp_send_channel = "openstack001.example.com" + $syslog = "192.168.0.10" + + # THIS STUFF ISN'T IN RELEASE YET + #$sentry_url = "http://192.168.0.19/sentry/store/" + #$sentry_key = "TODO:SENTRYPASS" + + $local_network = "192.168.0.0/16" + $vpn_ip = "192.168.0.2" + $public_interface = "eth0" + include novanode +# include nova-common + include opsmetrics + +# non-nova stuff such as nova-dash inherit from novanode +# novaspool needs a better home +# include novaspool +} + +# Builder +node "nova000.example.com" inherits novabase { + $syslog = "server" + include ntp + include syslog-server +} + +# Non-Nova nodes + +node + "blog.example.com", + "wiki.example.com" +inherits novabase { + include ganglia-python + include ganglia-apache + include ganglia-mysql +} + + +node "nova001.example.com" +inherits novabase { + include novabase + + nova_iptables { nova: + services => [ + "ganglia", + "mysql", + "rabbitmq", + "ldap", + "api", + "objectstore", + "nrpe", + ], + ip => "192.168.0.10", + } + + nobjectstore { nova: version => "0.9.0" } + nscheduler { nova: version => "0.9.0" } + napi { nova: + version => "0.9.0", + api_servers => 10, + api_base_port => 8000 + } +} + +node "nova002.example.com" +inherits novabase { + include novaconf + + nova_iptables { nova: + services => [ + "ganglia", + "dnsmasq", + "nrpe" + ], + ip => "192.168.4.2", + private_range => "192.168.0.0/16", + } + + nnetwork { nova: version => "0.9.0" } +} + +node + "nova003.example.com", + "nova004.example.com", + "nova005.example.com", + "nova006.example.com", + "nova007.example.com", + "nova008.example.com", + "nova009.example.com", + "nova010.example.com", + "nova011.example.com", + "nova012.example.com", + "nova013.example.com", + "nova014.example.com", + "nova015.example.com", + "nova016.example.com", + "nova017.example.com", + "nova018.example.com", + "nova019.example.com", +inherits novabase { + include novaconf + ncompute { nova: version => "0.9.0" } + nvolume { nova: version => "0.9.0" } +} + +#node +# "nova020.example.com" +# "nova021.example.com" +#inherits novanode { +# include novaconf + #ncompute { nova: version => "0.9.0" } +#} diff --git a/contrib/puppet/manifests/templates.pp b/contrib/puppet/manifests/templates.pp new file mode 100644 index 000000000000..90e433013878 --- /dev/null +++ b/contrib/puppet/manifests/templates.pp @@ -0,0 +1,21 @@ +# templates.pp + +import "classes/*" + +class baseclass { +# include dns-client # FIXME: missing resolv.conf.erb?? + include issue +} + +node default { + $nova_site = "undef" + $nova_ns1 = "undef" + $nova_ns2 = "undef" +# include baseclass +} + +# novanode handles the system-level requirements for Nova/Swift nodes +class novanode { + include baseclass + include lvmconf +} diff --git a/contrib/puppet/puppet.conf b/contrib/puppet/puppet.conf new file mode 100644 index 000000000000..92af920e34ef --- /dev/null +++ b/contrib/puppet/puppet.conf @@ -0,0 +1,11 @@ +[main] +logdir=/var/log/puppet +vardir=/var/lib/puppet +ssldir=/var/lib/puppet/ssl +rundir=/var/run/puppet +factpath=$vardir/lib/facter +pluginsync=false + +[puppetmasterd] +templatedir=/var/lib/nova/contrib/puppet/templates +autosign=true diff --git a/contrib/puppet/templates/haproxy.cfg.erb b/contrib/puppet/templates/haproxy.cfg.erb new file mode 100644 index 000000000000..bd9991de7ebf --- /dev/null +++ b/contrib/puppet/templates/haproxy.cfg.erb @@ -0,0 +1,39 @@ +# this config needs haproxy-1.1.28 or haproxy-1.2.1 + +global + log 127.0.0.1 local0 + log 127.0.0.1 local1 notice + #log loghost local0 info + maxconn 4096 + #chroot /usr/share/haproxy + stats socket /var/run/haproxy.sock + user haproxy + group haproxy + daemon + #debug + #quiet + +defaults + log global + mode http + option httplog + option dontlognull + retries 3 + option redispatch + stats enable + stats uri /haproxy + maxconn 2000 + contimeout 5000 + clitimeout 50000 + srvtimeout 50000 + + +listen nova-api 0.0.0.0:8773 + option httpchk GET / HTTP/1.0\r\nHost:\ example.com + option forwardfor + reqidel ^X-Forwarded-For:.* + balance roundrobin +<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset -%> + server api_<%= port %> 127.0.0.1:<%= port %> maxconn 1 check +<% end -%> + option httpclose # disable keep-alive diff --git a/contrib/puppet/templates/monitrc-nova-api.erb b/contrib/puppet/templates/monitrc-nova-api.erb new file mode 100644 index 000000000000..fe26263271ea --- /dev/null +++ b/contrib/puppet/templates/monitrc-nova-api.erb @@ -0,0 +1,138 @@ +############################################################################### +## Monit control file +############################################################################### +## +## Comments begin with a '#' and extend through the end of the line. Keywords +## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. +## +## Below you will find examples of some frequently used statements. For +## information about the control file, a complete list of statements and +## options please have a look in the monit manual. +## +## +############################################################################### +## Global section +############################################################################### +## +## Start monit in the background (run as a daemon): +# +set daemon 60 # check services at 1-minute intervals + with start delay 30 # optional: delay the first check by half a minute + # (by default check immediately after monit start) + + +## Set syslog logging with the 'daemon' facility. If the FACILITY option is +## omitted, monit will use 'user' facility by default. If you want to log to +## a stand alone log file instead, specify the path to a log file +# +set logfile syslog facility log_daemon +# +# +### Set the location of monit id file which saves the unique id specific for +### given monit. The id is generated and stored on first monit start. +### By default the file is placed in $HOME/.monit.id. +# +# set idfile /var/.monit.id +# +### Set the location of monit state file which saves the monitoring state +### on each cycle. By default the file is placed in $HOME/.monit.state. If +### state file is stored on persistent filesystem, monit will recover the +### monitoring state across reboots. If it is on temporary filesystem, the +### state will be lost on reboot. +# +# set statefile /var/.monit.state +# +## Set the list of mail servers for alert delivery. Multiple servers may be +## specified using comma separator. By default monit uses port 25 - this +## is possible to override with the PORT option. +# +# set mailserver mail.bar.baz, # primary mailserver +# backup.bar.baz port 10025, # backup mailserver on port 10025 +# localhost # fallback relay +# +# +## By default monit will drop alert events if no mail servers are available. +## If you want to keep the alerts for a later delivery retry, you can use the +## EVENTQUEUE statement. The base directory where undelivered alerts will be +## stored is specified by the BASEDIR option. You can limit the maximal queue +## size using the SLOTS option (if omitted, the queue is limited by space +## available in the back end filesystem). +# +# set eventqueue +# basedir /var/monit # set the base directory where events will be stored +# slots 100 # optionaly limit the queue size +# +# +## Send status and events to M/Monit (Monit central management: for more +## informations about M/Monit see http://www.tildeslash.com/mmonit). +# +# set mmonit http://monit:monit@192.168.1.10:8080/collector +# +# +## Monit by default uses the following alert mail format: +## +## --8<-- +## From: monit@$HOST # sender +## Subject: monit alert -- $EVENT $SERVICE # subject +## +## $EVENT Service $SERVICE # +## # +## Date: $DATE # +## Action: $ACTION # +## Host: $HOST # body +## Description: $DESCRIPTION # +## # +## Your faithful employee, # +## monit # +## --8<-- +## +## You can override this message format or parts of it, such as subject +## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. +## are expanded at runtime. For example, to override the sender: +# +# set mail-format { from: monit@foo.bar } +# +# +## You can set alert recipients here whom will receive alerts if/when a +## service defined in this file has errors. Alerts may be restricted on +## events by using a filter as in the second example below. +# +# set alert sysadm@foo.bar # receive all alerts +# set alert manager@foo.bar only on { timeout } # receive just service- +# # timeout alert +# +# +## Monit has an embedded web server which can be used to view status of +## services monitored, the current configuration, actual services parameters +## and manage services from a web interface. +# + set httpd port 2812 and + use address localhost # only accept connection from localhost + allow localhost # allow localhost to connect to the server and +# allow admin:monit # require user 'admin' with password 'monit' +# allow @monit # allow users of group 'monit' to connect (rw) +# allow @users readonly # allow users of group 'users' to connect readonly +# +# +############################################################################### +## Services +############################################################################### + +<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset %> + +check process nova_api_<%= port %> with pidfile /var/run/nova/nova-api-<%= port %>.pid + group nova_api + start program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock start" + as uid nova + stop program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock stop" + as uid nova + if failed port <%= port %> protocol http + with timeout 15 seconds + for 4 cycles + then restart + if totalmem > 300 Mb then restart + if cpu is greater than 60% for 2 cycles then alert + if cpu > 80% for 3 cycles then restart + if 3 restarts within 5 cycles then timeout + +<% end %> diff --git a/contrib/puppet/templates/nova-iptables.erb b/contrib/puppet/templates/nova-iptables.erb new file mode 100644 index 000000000000..2fc0663055f9 --- /dev/null +++ b/contrib/puppet/templates/nova-iptables.erb @@ -0,0 +1,10 @@ +<% services.each do |service| -%> +<%= service.upcase %>=1 +<% end -%> +<% if ip && ip != "" %>IP="<%=ip%>"<% end %> +<% if private_range && private_range != "" %>PRIVATE_RANGE="<%=private_range%>"<% end %> +<% if mgmt_ip && mgmt_ip != "" %>MGMT_IP="<%=mgmt_ip%>"<% end %> +<% if dmz_ip && dmz_ip != "" %>DMZ_IP="<%=dmz_ip%>"<% end %> + +# warning: this file is auto-generated by puppet + diff --git a/contrib/puppet/templates/production/nova-common.conf.erb b/contrib/puppet/templates/production/nova-common.conf.erb new file mode 100644 index 000000000000..00c110781941 --- /dev/null +++ b/contrib/puppet/templates/production/nova-common.conf.erb @@ -0,0 +1,56 @@ +# global +--dmz_net=192.168.0.0 +--dmz_mask=255.255.0.0 +--dmz_cidr=192.168.0.0/16 +--ldap_user_dn=cn=Administrators,dc=example,dc=com +--ldap_user_unit=Users +--ldap_user_subtree=ou=Users,dc=example,dc=com +--ldap_project_subtree=ou=Groups,dc=example,dc=com +--role_project_subtree=ou=Groups,dc=example,dc=com +--ldap_cloudadmin=cn=NovaAdmins,ou=Groups,dc=example,dc=com +--ldap_itsec=cn=NovaSecurity,ou=Groups,dc=example,dc=com +--ldap_sysadmin=cn=Administrators,ou=Groups,dc=example,dc=com +--ldap_netadmin=cn=Administrators,ou=Groups,dc=example,dc=com +--ldap_developer=cn=developers,ou=Groups,dc=example,dc=com +--verbose +--daemonize +--syslog +--networks_path=/var/lib/nova/networks +--instances_path=/var/lib/nova/instances +--buckets_path=/var/lib/nova/objectstore/buckets +--images_path=/var/lib/nova/objectstore/images +--scheduler_driver=nova.scheduler.simple.SimpleScheduler +--libvirt_xml_template=/usr/share/nova/libvirt.qemu.xml.template +--credentials_template=/usr/share/nova/novarc.template +--boot_script_template=/usr/share/nova/bootscript.template +--vpn_client_template=/usr/share/nova/client.ovpn.template +--max_cores=40 +--max_gigabytes=2000 +--ca_path=/var/lib/nova/CA +--keys_path=/var/lib/nova/keys +--vpn_start=11000 +--volume_group=vgdata +--volume_manager=nova.volume.manager.ISCSIManager +--volume_driver=nova.volume.driver.ISCSIDriver +--default_kernel=aki-DEFAULT +--default_ramdisk=ari-DEFAULT +--dhcpbridge=/usr/bin/nova-dhcpbridge +--vpn_image_id=ami-cloudpipe +--dhcpbridge_flagfile=/etc/nova/nova.conf +--credential_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=NOVA/CN=%s-%s +--auth_driver=nova.auth.ldapdriver.LdapDriver +--quota_cores=17 +--quota_floating_ips=5 +--quota_instances=6 +--quota_volumes=10 +--quota_gigabytes=100 +--use_nova_chains=True +--input_chain=services +--FAKE_subdomain=ec2 +--use_project_ca=True +--fixed_ip_disassociate_timeout=300 +--api_max_requests=1 +--api_listen_ip=127.0.0.1 +--user_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=%s-%s-%s +--project_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-ca-%s-%s +--vpn_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-vpn-%s-%s diff --git a/contrib/puppet/templates/production/nova-nova.conf.erb b/contrib/puppet/templates/production/nova-nova.conf.erb new file mode 100644 index 000000000000..8683fefde694 --- /dev/null +++ b/contrib/puppet/templates/production/nova-nova.conf.erb @@ -0,0 +1,21 @@ +--fixed_range=192.168.0.0/16 +--iscsi_ip_prefix=192.168.4 +--floating_range=10.0.0.0/24 +--rabbit_host=192.168.0.10 +--s3_host=192.168.0.10 +--cc_host=192.168.0.10 +--cc_dmz=192.168.24.10 +--s3_dmz=192.168.24.10 +--ec2_url=http://192.168.0.1:8773/services/Cloud +--vpn_ip=192.168.0.2 +--ldap_url=ldap://192.168.0.10 +--sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova +--other_sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova +--routing_source_ip=192.168.0.2 +--bridge_dev=eth1 +--public_interface=eth0 +--vlan_start=3100 +--num_networks=700 +--rabbit_userid=TODO:RABBIT +--rabbit_password=TODO:CHANGEME +--ldap_password=TODO:CHANGEME diff --git a/doc/source/Makefile b/doc/Makefile similarity index 93% rename from doc/source/Makefile rename to doc/Makefile index b2f74e85aaf0..251a008e0e33 100644 --- a/doc/source/Makefile +++ b/doc/Makefile @@ -4,16 +4,19 @@ # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build +SPHINXSOURCE = source PAPER = -BUILDDIR = _build +BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(SPHINXSOURCE) .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest +.DEFAULT_GOAL = html + help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @@ -29,6 +32,11 @@ help: clean: -rm -rf $(BUILDDIR)/* + -rm -rf nova.sqlite + if [ -f .autogenerated ] ; then \ + cat .autogenerated | xargs rm ; \ + rm .autogenerated ; \ + fi html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/doc/README.rst b/doc/README.rst new file mode 100644 index 000000000000..565e8d5bf5fd --- /dev/null +++ b/doc/README.rst @@ -0,0 +1,33 @@ +================= +Building the docs +================= + +It is really easy. You'll need sphinx (the python one) and if you are using the virtualenv you'll need to install it in the virtualenv specifically so that it can load the nova modules. + + +Use `make` +========== + +Just type make:: + + % make + +Look in the Makefile for more targets. + + +Manually +======== + + 1. Generate the code.rst file so that Sphinx will pull in our docstrings:: + + % ./generate_autodoc_index.sh > source/code.rst + + 2. Run `sphinx_build`:: + + % sphinx-build -b html source build/html + + +The docs have been built +======================== + +Check out the `build` directory to find them. Yay! diff --git a/doc/ext/__init__.py b/doc/ext/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/doc/ext/nova_autodoc.py b/doc/ext/nova_autodoc.py new file mode 100644 index 000000000000..39aa2c2cfadd --- /dev/null +++ b/doc/ext/nova_autodoc.py @@ -0,0 +1,9 @@ +import os + +from nova import utils + +def setup(app): + rootdir = os.path.abspath(app.srcdir + '/..') + print "**Autodocumenting from %s" % rootdir + rv = utils.execute('cd %s && ./generate_autodoc_index.sh' % rootdir) + print rv[0] diff --git a/doc/ext/nova_todo.py b/doc/ext/nova_todo.py new file mode 100644 index 000000000000..efc0c3edd6f8 --- /dev/null +++ b/doc/ext/nova_todo.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# This is a hack of the builtin todo extension, to make the todo_list more user friendly + +from sphinx.ext.todo import * +from docutils.parsers.rst import directives +import re + +def _(s): + return s + + +def process_todo_nodes(app, doctree, fromdocname): + if not app.config['todo_include_todos']: + for node in doctree.traverse(todo_node): + node.parent.remove(node) + + # Replace all todolist nodes with a list of the collected todos. + # Augment each todo with a backlink to the original location. + env = app.builder.env + + if not hasattr(env, 'todo_all_todos'): + env.todo_all_todos = [] + + + # remove the item that was added in the constructor, since I'm tired of + # reading through docutils for the proper way to construct an empty list + lists = [] + for i in xrange(5): + lists.append(nodes.bullet_list("", nodes.Text('',''))); + lists[i].remove(lists[i][0]) + lists[i].set_class('todo_list') + + for node in doctree.traverse(todolist): + if not app.config['todo_include_todos']: + node.replace_self([]) + continue + + for todo_info in env.todo_all_todos: + para = nodes.paragraph() + filename = env.doc2path(todo_info['docname'], base=None) + + # Create a reference + newnode = nodes.reference('', '') + + link = _('%s, line %d') % (filename, todo_info['lineno']); + innernode = nodes.emphasis(link, link) + newnode['refdocname'] = todo_info['docname'] + + try: + newnode['refuri'] = app.builder.get_relative_uri( + fromdocname, todo_info['docname']) + newnode['refuri'] += '#' + todo_info['target']['refid'] + except NoUri: + # ignore if no URI can be determined, e.g. for LaTeX output + pass + + newnode.append(innernode) + para += newnode + para.set_class('todo_link') + + todo_entry = todo_info['todo'] + + env.resolve_references(todo_entry, todo_info['docname'], app.builder) + + item = nodes.list_item('', para) + todo_entry[1].set_class('details') + + comment = todo_entry[1] + + m = re.match(r"^P(\d)", comment.astext()) + priority = 5 + if m: + priority = int(m.group(1)) + if (priority < 0): priority = 1 + if (priority > 5): priority = 5 + + item.set_class('todo_p' + str(priority)) + todo_entry.set_class('todo_p' + str(priority)) + + item.append(comment) + + lists[priority-1].insert(0, item) + + + node.replace_self(lists) + +def setup(app): + app.add_config_value('todo_include_todos', False, False) + + app.add_node(todolist) + app.add_node(todo_node, + html=(visit_todo_node, depart_todo_node), + latex=(visit_todo_node, depart_todo_node), + text=(visit_todo_node, depart_todo_node)) + + app.add_directive('todo', Todo) + app.add_directive('todolist', TodoList) + app.connect('doctree-read', process_todos) + app.connect('doctree-resolved', process_todo_nodes) + app.connect('env-purge-doc', purge_todos) + diff --git a/doc/find_autodoc_modules.sh b/doc/find_autodoc_modules.sh new file mode 100755 index 000000000000..c612b0084068 --- /dev/null +++ b/doc/find_autodoc_modules.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +NOVA_DIR='../nova/' # include trailing slash +DOCS_DIR='source' + +modules='' +for x in `find ${NOVA_DIR} -name '*.py'`; do + if [ `basename ${x} .py` == "__init__" ] ; then + continue + fi + relative=nova.`echo ${x} | sed -e 's$^'${NOVA_DIR}'$$' -e 's/.py$//' -e 's$/$.$g'` + modules="${modules} ${relative}" +done + +for mod in ${modules} ; do + if [ ! -f "${DOCS_DIR}/${mod}.rst" ]; + then + echo ${mod} + fi +done diff --git a/doc/generate_autodoc_index.sh b/doc/generate_autodoc_index.sh new file mode 100755 index 000000000000..4e2870bfc929 --- /dev/null +++ b/doc/generate_autodoc_index.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +SOURCEDIR=source/api + +if [ ! -d ${SOURCEDIR} ] ; then + mkdir -p ${SOURCEDIR} +fi + +for x in `./find_autodoc_modules.sh`; +do + echo "Generating ${SOURCEDIR}/${x}.rst" + echo "${SOURCEDIR}/${x}.rst" >> .autogenerated + ( cat < ${SOURCEDIR}/${x}.rst + +done + +if [ ! -f ${SOURCEDIR}/autoindex.rst ] ; then + + cat > ${SOURCEDIR}/autoindex.rst <> ${SOURCEDIR}/autoindex.rst + done + + echo ${SOURCEDIR}/autoindex.rst >> .autogenerated +fi diff --git a/doc/source/_ga/layout.html b/doc/source/_ga/layout.html index 0b72a77ac291..f29e90968545 100644 --- a/doc/source/_ga/layout.html +++ b/doc/source/_ga/layout.html @@ -8,7 +8,7 @@ document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga. +{% endblock %} + +{%- macro sidebar() %} + {%- if not embedded %}{% if not theme_nosidebar|tobool %} +
+
+ {%- block sidebarlogo %} + {%- if logo %} + + {%- endif %} + {%- endblock %} + {%- block sidebartoc %} + {%- if display_toc %} +

{{ _('Table Of Contents') }}

+ {{ toc }} + {%- endif %} + {%- endblock %} + {%- block sidebarrel %} + {%- if prev %} +

{{ _('Previous topic') }}

+

{{ prev.title }}

+ {%- endif %} + {%- if next %} +

{{ _('Next topic') }}

+

{{ next.title }}

+ {%- endif %} + {%- endblock %} + {%- block sidebarsourcelink %} + {%- if show_source and has_source and sourcename %} +

{{ _('This Page') }}

+ + {%- endif %} + {%- endblock %} + {%- if customsidebar %} + {% include customsidebar %} + {%- endif %} + {%- block sidebarsearch %} + {%- if pagename != "search" %} + + + {%- endif %} + + {%- if pagename == "index" %} +

{{ _('Twitter Feed') }}

+ + {%- endif %} + + + {%- endblock %} +
+
+ {%- endif %}{% endif %} +{%- endmacro %} diff --git a/doc/source/_theme/theme.conf b/doc/source/_theme/theme.conf new file mode 100644 index 000000000000..e039fe01f99d --- /dev/null +++ b/doc/source/_theme/theme.conf @@ -0,0 +1,5 @@ +[theme] +inherit = sphinxdoc +stylesheet = sphinxdoc.css +pygments_style = friendly + diff --git a/doc/source/adminguide/binaries.rst b/doc/source/adminguide/binaries.rst new file mode 100644 index 000000000000..25605adf94fc --- /dev/null +++ b/doc/source/adminguide/binaries.rst @@ -0,0 +1,57 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _binaries: + +Nova Daemons +============= + +The configuration of these binaries relies on "flagfiles" using the google +gflags package:: + + $ nova-xxxxx --flagfile flagfile + +The binaries can all run on the same machine or be spread out amongst multiple boxes in a large deployment. + +nova-api +-------- + +Nova api receives xml requests and sends them to the rest of the system. It is a wsgi app that routes and authenticate requests. It supports the ec2 and openstack apis. + +nova-objectstore +---------------- + +Nova objectstore is an ultra simple file-based storage system for images that replicates most of the S3 Api. It will soon be replaced with glance and a simple image manager. + +nova-compute +------------ + +Nova compute is responsible for managing virtual machines. It loads a Service object which exposes the public methods on ComputeManager via rpc. + +nova-volume +----------- + +Nova volume is responsible for managing attachable block storage devices. It loads a Service object which exposes the public methods on VolumeManager via rpc. + +nova-network +------------ + +Nova network is responsible for managing floating and fixed ips, dhcp, bridging and vlans. It loads a Service object which exposes the public methods on one of the subclasses of NetworkManager. Different networking strategies are as simple as changing the network_manager flag:: + + $ nova-network --network_manager=nova.network.manager.FlatManager + +IMPORTANT: Make sure that you also set the network_manager on nova-api and nova_compute, since make some calls to network manager in process instead of through rpc. More information on the interactions between services, managers, and drivers can be found :ref:`here ` diff --git a/doc/source/adminguide/distros/others.rst b/doc/source/adminguide/distros/others.rst new file mode 100644 index 000000000000..ec14a9abb2b1 --- /dev/null +++ b/doc/source/adminguide/distros/others.rst @@ -0,0 +1,88 @@ +Installation on other distros (like Debian, Fedora or CentOS ) +============================================================== + +Feel free to add additional notes for additional distributions. + +Nova installation on CentOS 5.5 +------------------------------- + +These are notes for installing OpenStack Compute on CentOS 5.5 and will be updated but are NOT final. Please test for accuracy and edit as you see fit. + +The principle botleneck for running nova on centos in python 2.6. Nova is written in python 2.6 and CentOS 5.5. comes with python 2.4. We can not update python system wide as some core utilities (like yum) is dependent on python 2.4. Also very few python 2.6 modules are available in centos/epel repos. + +Pre-reqs +-------- + +Add euca2ools and EPEL repo first.:: + + cat >/etc/yum.repos.d/euca2ools.repo << EUCA_REPO_CONF_EOF + [eucalyptus] + name=euca2ools + baseurl=http://www.eucalyptussoftware.com/downloads/repo/euca2ools/1.3.1/yum/centos/ + enabled=1 + gpgcheck=0 + + EUCA_REPO_CONF_EOF + +:: + + rpm -Uvh 'http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-4.noarch.rpm' + +Now install python2.6, kvm and few other libraries through yum:: + + yum -y install dnsmasq vblade kpartx kvm gawk iptables ebtables bzr screen euca2ools curl rabbitmq-server gcc gcc-c++ autoconf automake swig openldap openldap-servers nginx python26 python26-devel python26-distribute git openssl-devel python26-tools mysql-server qemu kmod-kvm libxml2 libxslt libxslt-devel mysql-devel + +Then download the latest aoetools and then build(and install) it, check for the latest version on sourceforge, exact url will change if theres a new release:: + + wget -c http://sourceforge.net/projects/aoetools/files/aoetools/32/aoetools-32.tar.gz/download + tar -zxvf aoetools-32.tar.gz + cd aoetools-32 + make + make install + +Add the udev rules for aoetools:: + + cat > /etc/udev/rules.d/60-aoe.rules << AOE_RULES_EOF + SUBSYSTEM=="aoe", KERNEL=="discover", NAME="etherd/%k", GROUP="disk", MODE="0220" + SUBSYSTEM=="aoe", KERNEL=="err", NAME="etherd/%k", GROUP="disk", MODE="0440" + SUBSYSTEM=="aoe", KERNEL=="interfaces", NAME="etherd/%k", GROUP="disk", MODE="0220" + SUBSYSTEM=="aoe", KERNEL=="revalidate", NAME="etherd/%k", GROUP="disk", MODE="0220" + # aoe block devices + KERNEL=="etherd*", NAME="%k", GROUP="disk" + AOE_RULES_EOF + +Load the kernel modules:: + + modprobe aoe + +:: + + modprobe kvm + +Now, install the python modules using easy_install-2.6, this ensures the installation are done against python 2.6 + + +easy_install-2.6 twisted sqlalchemy mox greenlet carrot daemon eventlet tornado IPy routes lxml MySQL-python +python-gflags need to be downloaded and installed manually, use these commands (check the exact url for newer releases ): + +:: + + wget -c "http://python-gflags.googlecode.com/files/python-gflags-1.4.tar.gz" + tar -zxvf python-gflags-1.4.tar.gz + cd python-gflags-1.4 + python2.6 setup.py install + cd .. + +Same for python2.6-libxml2 module, notice the --with-python and --prefix flags. --with-python ensures we are building it against python2.6 (otherwise it will build against python2.4, which is default):: + + wget -c "ftp://xmlsoft.org/libxml2/libxml2-2.7.3.tar.gz" + tar -zxvf libxml2-2.7.3.tar.gz + cd libxml2-2.7.3 + ./configure --with-python=/usr/bin/python26 --prefix=/usr + make all + make install + cd python + python2.6 setup.py install + cd .. + +Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/distros/ubuntu.10.04.rst b/doc/source/adminguide/distros/ubuntu.10.04.rst new file mode 100644 index 000000000000..ce368fab8ddc --- /dev/null +++ b/doc/source/adminguide/distros/ubuntu.10.04.rst @@ -0,0 +1,41 @@ +Installing on Ubuntu 10.04 (Lucid) +================================== + +Step 1: Install dependencies +---------------------------- +Grab the latest code from launchpad: + +:: + + bzr clone lp:nova + +Here's a script you can use to install (and then run) Nova on Ubuntu or Debian (when using Debian, edit nova.sh to have USE_PPA=0): + +.. todo:: give a link to a stable releases page + +Step 2: Install dependencies +---------------------------- + +Nova requires rabbitmq for messaging and optionally you can use redis for storing state, so install these first. + +*Note:* You must have sudo installed to run these commands as shown here. + +:: + + sudo apt-get install rabbitmq-server redis-server + + +You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. + +If you're running on Ubuntu 10.04, you'll need to install Twisted and python-gflags which is included in the OpenStack PPA. + +:: + + sudo apt-get install python-twisted + + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 95C71FE2 + sudo sh -c 'echo "deb http://ppa.launchpad.net/openstack/openstack-ppa/ubuntu lucid main" > /etc/apt/sources.list.d/openstackppa.list' + sudo apt-get update && sudo apt-get install python-gflags + + +Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/distros/ubuntu.10.10.rst b/doc/source/adminguide/distros/ubuntu.10.10.rst new file mode 100644 index 000000000000..a3fa2def1ed6 --- /dev/null +++ b/doc/source/adminguide/distros/ubuntu.10.10.rst @@ -0,0 +1,41 @@ +Installing on Ubuntu 10.10 (Maverick) +===================================== +Single Machine Installation (Ubuntu 10.10) + +While we wouldn't expect you to put OpenStack Compute into production on a non-LTS version of Ubuntu, these instructions are up-to-date with the latest version of Ubuntu. + +Make sure you are running Ubuntu 10.10 so that the packages will be available. This install requires more than 70 MB of free disk space. + +These instructions are based on Soren Hansen's blog entry, Openstack on Maverick. A script is in progress as well. + +Step 1: Install required prerequisites +-------------------------------------- +Nova requires rabbitmq for messaging and redis for storing state (for now), so we'll install these first.:: + + sudo apt-get install rabbitmq-server redis-server + +You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. + +Step 2: Install Nova packages available in Maverick Meerkat +----------------------------------------------------------- +Type or copy/paste in the following line to get the packages that you use to run OpenStack Compute.:: + + sudo apt-get install python-nova + sudo apt-get install nova-api nova-objectstore nova-compute nova-scheduler nova-network euca2ools unzip + +You'll see messages starting with "Reading package lists... Done" and you must confirm by typing Y that you want to continue. This operation may take a while as many dependent packages will be installed. Note: there is a dependency problem with python-nova which can be worked around by installing first. + +When the installation is complete, you'll see the following lines confirming::: + + Adding system user `nova' (UID 106) ... + Adding new user `nova' (UID 106) with group `nogroup' ... + Not creating home directory `/var/lib/nova'. + Setting up nova-scheduler (0.9.1~bzr331-0ubuntu2) ... + * Starting nova scheduler nova-scheduler + WARNING:root:Starting scheduler node + ...done. + Processing triggers for libc-bin ... + ldconfig deferred processing now taking place + Processing triggers for python-support ... + +Once you've done this, continue at Step 3 here: :doc:`../single.node.install` diff --git a/doc/source/adminguide/euca2ools.rst b/doc/source/adminguide/euca2ools.rst new file mode 100644 index 000000000000..6f0c57358837 --- /dev/null +++ b/doc/source/adminguide/euca2ools.rst @@ -0,0 +1,49 @@ +Euca2ools +========= + +Nova is compatible with most of the euca2ools command line utilities. Both Administrators and Users will find these tools helpful for day-to-day administration. + +* euca-add-group +* euca-delete-bundle +* euca-describe-instances +* euca-register +* euca-add-keypair +* euca-delete-group +* euca-describe-keypairs +* euca-release-address +* euca-allocate-address +* euca-delete-keypair +* euca-describe-regions +* euca-reset-image-attribute +* euca-associate-address +* euca-delete-snapshot +* euca-describe-snapshots +* euca-revoke +* euca-attach-volume +* euca-delete-volume +* euca-describe-volumes +* euca-run-instances +* euca-authorize +* euca-deregister +* euca-detach-volume +* euca-terminate-instances +* euca-bundle-image +* euca-describe-addresses +* euca-disassociate-address +* euca-unbundle +* euca-bundle-vol +* euca-describe-availability-zones +* euca-download-bundle +* euca-upload-bundle +* euca-confirm-product-instance +* euca-describe-groups +* euca-get-console-output +* euca-version +* euca-create-snapshot +* euca-describe-image-attribute +* euca-modify-image-attribute +* euca-create-volume +* euca-describe-images +* euca-reboot-instances + + diff --git a/doc/source/packages.rst b/doc/source/adminguide/flags.rst similarity index 66% rename from doc/source/packages.rst rename to doc/source/adminguide/flags.rst index 6029ad7d749b..4c950aa88626 100644 --- a/doc/source/packages.rst +++ b/doc/source/adminguide/flags.rst @@ -15,15 +15,9 @@ License for the specific language governing permissions and limitations under the License. -nova Packages & Dependencies -============================ +Flags and Flagfiles +=================== -Nova is being built on Ubuntu Lucid. - -The following packages are required: - - apt-get install python-ipy, python-libvirt, python-boto, python-pycurl, python-twisted, python-daemon, python-redis, python-carrot, python-lockfile - -In addition you need to install python: - - * python-gflags - http://code.google.com/p/python-gflags/ +* python-gflags +* flagfiles +* list of flags by component (see concepts list) diff --git a/doc/source/adminguide/getting.started.rst b/doc/source/adminguide/getting.started.rst new file mode 100644 index 000000000000..7075a0b02f9d --- /dev/null +++ b/doc/source/adminguide/getting.started.rst @@ -0,0 +1,168 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Getting Started with Nova +========================= + +This code base is continually changing, so dependencies also change. If you +encounter any problems, see the :doc:`../community` page. +The `contrib/nova.sh` script should be kept up to date, and may be a good +resource to review when debugging. + +The purpose of this document is to get a system installed that you can use to +test your setup assumptions. Working from this base installtion you can +tweak configurations and work with different flags to monitor interaction with +your hardware, network, and other factors that will allow you to determine +suitability for your deployment. After following this setup method, you should +be able to experiment with different managers, drivers, and flags to get the +best performance. + +Dependencies +------------ + +Related servers we rely on + +* **RabbitMQ**: messaging queue, used for all communication between components + +Optional servers + +* **OpenLDAP**: By default, the auth server uses the RDBMS-backed datastore by + setting FLAGS.auth_driver to `nova.auth.dbdriver.DbDriver`. But OpenLDAP + (or LDAP) could be configured by specifying `nova.auth.ldapdriver.LdapDriver`. + There is a script in the sources (`nova/auth/slap.sh`) to install a very basic + openldap server on ubuntu. +* **ReDIS**: There is a fake ldap auth driver + `nova.auth.ldapdriver.FakeLdapDriver` that backends to redis. This was + created for testing ldap implementation on systems that don't have an easy + means to install ldap. +* **MySQL**: Either MySQL or another database supported by sqlalchemy needs to + be avilable. Currently, only sqlite3 an mysql have been tested. + +Python libraries that we use (from pip-requires): + +.. literalinclude:: ../../../tools/pip-requires + +Other libraries: + +* **XenAPI**: Needed only for Xen Cloud Platform or XenServer support. Available + from http://wiki.xensource.com/xenwiki/XCP_SDK or + http://community.citrix.com/cdn/xs/sdks. + +External unix tools that are required: + +* iptables +* ebtables +* gawk +* curl +* kvm +* libvirt +* dnsmasq +* vlan +* open-iscsi and iscsitarget (if you use iscsi volumes) +* aoetools and vblade-persist (if you use aoe-volumes) + +Nova uses cutting-edge versions of many packages. There are ubuntu packages in +the nova-core ppa. You can use add this ppa to your sources list on an ubuntu +machine with the following commands:: + + sudo apt-get install -y python-software-properties + sudo add-apt-repository ppa:nova-core/ppa + +Recommended +----------- + +* euca2ools: python implementation of aws ec2-tools and ami tools +* build tornado to use C module for evented section + + +Installation +-------------- + +You can install from packages for your particular Linux distribution if they are +available. Otherwise you can install from source by checking out the source +files from the `Nova Source Code Repository `_ +and running:: + + python setup.py install + +Configuration +--------------- + +Configuring the host system +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As you read through the Administration Guide you will notice configuration hints +inline with documentation on the subsystem you are configuring. Presented in +this "Getting Started with Nova" document, we only provide what you need to +get started as quickly as possible. For a more detailed description of system +configuration, start reading through :doc:`multi.node.install`. + +* Create a volume group (you can use an actual disk for the volume group as + well):: + + # This creates a 1GB file to create volumes out of + dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 + losetup --show -f MY_FILE_PATH + # replace /dev/loop0 below with whatever losetup returns + # nova-volumes is the default for the --volume_group flag + vgcreate nova-volumes /dev/loop0 + + +Configuring Nova +~~~~~~~~~~~~~~~~ + +Configuration of the entire system is performed through python-gflags. The +best way to track configuration is through the use of a flagfile. + +A flagfile is specified with the ``--flagfile=FILEPATH`` argument to the binary +when you launch it. Flagfiles for nova are typically stored in +``/etc/nova/nova.conf``, and flags specific to a certain program are stored in +``/etc/nova/nova-COMMAND.conf``. Each configuration file can include another +flagfile, so typically a file like ``nova-manage.conf`` would have as its first +line ``--flagfile=/etc/nova/nova.conf`` to load the common flags before +specifying overrides or additional options. + +A sample configuration to test the system follows:: + + --verbose + --nodaemon + --FAKE_subdomain=ec2 + --auth_driver=nova.auth.dbdriver.DbDriver + +Running +--------- + +There are many parts to the nova system, each with a specific function. They +are built to be highly-available, so there are may configurations they can be +run in (ie: on many machines, many listeners per machine, etc). This part +of the guide only gets you started quickly, to learn about HA options, see +:doc:`multi.node.install`. + +Launch supporting services + +* rabbitmq +* redis (optional) +* mysql (optional) +* openldap (optional) + +Launch nova components, each should have ``--flagfile=/etc/nova/nova.conf`` + +* nova-api +* nova-compute +* nova-objectstore +* nova-volume +* nova-scheduler diff --git a/doc/source/adminguide/index.rst b/doc/source/adminguide/index.rst new file mode 100644 index 000000000000..51228b3191cc --- /dev/null +++ b/doc/source/adminguide/index.rst @@ -0,0 +1,90 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Administration Guide +==================== + +This guide describes the basics of running and managing Nova. + +Running the Cloud +----------------- + +The fastest way to get a test cloud running is by following the directions in the :doc:`../quickstart`. + +Nova's cloud works via the interaction of a series of daemon processes that reside persistently on the host machine(s). Fortunately, the :doc:`../quickstart` process launches sample versions of all these daemons for you. Once you are familiar with basic Nova usage, you can learn more about daemons by reading :doc:`../service.architecture` and :doc:`binaries`. + +Administration Utilities +------------------------ + +There are two main tools that a system administrator will find useful to manage their Nova cloud: + +.. toctree:: + :maxdepth: 1 + + nova.manage + euca2ools + +nova-manage may only be run by users with admin priviledges. euca2ools can be used by all users, though specific commands may be restricted by Role Based Access Control. You can read more about creating and managing users in :doc:`managing.users` + +User and Resource Management +---------------------------- + +nova-manage and euca2ools provide the basic interface to perform a broad range of administration functions. In this section, you can read more about how to accomplish specific administration tasks. + +For background on the core objects refenced in this section, see :doc:`../object.model` + +.. toctree:: + :maxdepth: 1 + + managing.users + managing.projects + managing.instances + managing.images + managing.volumes + managing.networks + +Deployment +---------- + +.. todo:: talk about deployment scenarios + +.. toctree:: + :maxdepth: 1 + + multi.node.install + + +Networking +^^^^^^^^^^ + +.. toctree:: + :maxdepth: 1 + + multi.node.install + network.vlan.rst + network.flat.rst + + +Advanced Topics +--------------- + +.. toctree:: + :maxdepth: 1 + + flags + monitoring + diff --git a/doc/source/binaries.rst b/doc/source/adminguide/managing.images.rst similarity index 67% rename from doc/source/binaries.rst rename to doc/source/adminguide/managing.images.rst index 90a9581f7f8d..df71db23b91b 100644 --- a/doc/source/binaries.rst +++ b/doc/source/adminguide/managing.images.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,17 +15,7 @@ License for the specific language governing permissions and limitations under the License. -Nova Binaries +Managing Images =============== -* nova-api -* nova-compute -* nova-manage -* nova-objectstore -* nova-volume - -The configuration of these binaries relies on "flagfiles" using the google -gflags package. If present, the nova.conf file will be used as the flagfile -- otherwise, it must be specified on the command line:: - - $ python node_worker.py --flagfile flagfile +.. todo:: Put info on managing images here! diff --git a/doc/source/adminguide/managing.instances.rst b/doc/source/adminguide/managing.instances.rst new file mode 100644 index 000000000000..d97567bb2c58 --- /dev/null +++ b/doc/source/adminguide/managing.instances.rst @@ -0,0 +1,59 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Instances +================== + +Keypairs +-------- + +Images can be shared by many users, so it is dangerous to put passwords into the images. Nova therefore supports injecting ssh keys into instances before they are booted. This allows a user to login to the instances that he or she creates securely. Generally the first thing that a user does when using the system is create a keypair. Nova generates a public and private key pair, and sends the private key to the user. The public key is stored so that it can be injected into instances. + +Keypairs are created through the api. They can be created on the command line using the euca2ools script euca-add-keypair. Refer to the man page for the available options. Example usage:: + + euca-add-keypair test > test.pem + chmod 600 test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + # wait for boot + ssh -i test.pem root@ip.of.instance + + +Basic Management +---------------- +Instance management can be accomplished with euca commands: + + +To run an instance: + +:: + + euca-run-instances + + +To terminate an instance: + +:: + + euca-terminate-instances + +To reboot an instance: + +:: + + euca-reboot-instances + +See the euca2ools documentation for more information diff --git a/doc/source/adminguide/managing.networks.rst b/doc/source/adminguide/managing.networks.rst new file mode 100644 index 000000000000..c8df471e84ad --- /dev/null +++ b/doc/source/adminguide/managing.networks.rst @@ -0,0 +1,85 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + Overview Sections Copyright 2010 Citrix + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Networking Overview +=================== +In Nova, users organize their cloud resources in projects. A Nova project consists of a number of VM instances created by a user. For each VM instance, Nova assigns to it a private IP address. (Currently, Nova only supports Linux bridge networking that allows the virtual interfaces to connect to the outside network through the physical interface. Other virtual network technologies, such as Open vSwitch, could be supported in the future.) The Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. + +.. + (perhaps some of this should be moved elsewhere) + Introduction + ------------ + + Nova consists of seven main components, with the Cloud Controller component representing the global state and interacting with all other components. API Server acts as the Web services front end for the cloud controller. Compute Controller provides compute server resources, and the Object Store component provides storage services. Auth Manager provides authentication and authorization services. Volume Controller provides fast and permanent block-level storage for the comput servers. Network Controller provides virtual networks to enable compute servers to interact with each other and with the public network. Scheduler selects the most suitable compute controller to host an instance. + + .. todo:: Insert Figure 1 image from "An OpenStack Network Overview" contributed by Citrix + + Nova is built on a shared-nothing, messaging-based architecture. All of the major components, that is Compute Controller, Volume Controller, Network Controller, and Object Store can be run on multiple servers. Cloud Controller communicates with Object Store via HTTP (Hyper Text Transfer Protocol), but it communicates with Scheduler, Network Controller, and Volume Controller via AMQP (Advanced Message Queue Protocol). To avoid blocking each component while waiting for a response, Nova uses asynchronous calls, with a call-back that gets triggered when a response is received. + + To achieve the shared-nothing property with multiple copies of the same component, Nova keeps all the cloud system state in a distributed data store. Updates to system state are written into this store, using atomic transactions when required. Requests for system state are read out of this store. In limited cases, the read results are cached within controllers for short periods of time (for example, the current list of system users.) + + .. note:: The database schema is available on the `OpenStack Wiki _`. + +Nova Network Strategies +----------------------- + +Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can c-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. + +Read more about Nova network strategies here: + +.. toctree:: + :maxdepth: 1 + + network.flat.rst + network.vlan.rst + + +Network Management Commands +--------------------------- + +Admins and Network Administrators can use the 'nova-manage' command to manage network resources: + +VPN Management +~~~~~~~~~~~~~~ + +* vpn list: Print a listing of the VPNs for all projects. + * arguments: none +* vpn run: Start the VPN for a given project. + * arguments: project +* vpn spawn: Run all VPNs. + * arguments: none + + +Floating IP Management +~~~~~~~~~~~~~~~~~~~~~~ + +* floating create: Creates floating ips for host by range + * arguments: host ip_range +* floating delete: Deletes floating ips by range + * arguments: range +* floating list: Prints a listing of all floating ips + * arguments: none + +Network Management +~~~~~~~~~~~~~~~~~~ + +* network create: Creates fixed ips for host by range + * arguments: [fixed_range=FLAG], [num_networks=FLAG], + [network_size=FLAG], [vlan_start=FLAG], + [vpn_start=FLAG] + diff --git a/doc/source/adminguide/managing.projects.rst b/doc/source/adminguide/managing.projects.rst new file mode 100644 index 000000000000..b592e14d77d4 --- /dev/null +++ b/doc/source/adminguide/managing.projects.rst @@ -0,0 +1,68 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Managing Projects +================= + +Projects are isolated resource containers forming the principal organizational structure within Nova. They consist of a separate vlan, volumes, instances, images, keys, and users. + +Although the original ec2 api only supports users, nova adds the concept of projects. A user can specify which project he or she wishes to use by appending `:project_id` to his or her access key. If no project is specified in the api request, nova will attempt to use a project with the same id as the user. + +The api will return NotAuthorized if a normal user attempts to make requests for a project that he or she is not a member of. Note that admins or users with special admin roles skip this check and can make requests for any project. + +To create a project, use the `project create` command of nova-manage. The syntax is nova-manage project create projectname manager_id [description] You must specify a projectname and a manager_id. For example:: + nova-manage project create john_project john "This is a sample project" + +You can add and remove users from projects with `project add` and `project remove`:: + nova-manage project add john_project john + nova-manage project remove john_project john + +Project Commands +---------------- + +Admins and Project Managers can use the 'nova-manage project' command to manage project resources: + +* project add: Adds user to project + * arguments: project user +* project create: Creates a new project + * arguments: name project_manager [description] +* project delete: Deletes an existing project + * arguments: project_id +* project environment: Exports environment variables to an sourcable file + * arguments: project_id user_id [filename='novarc] +* project list: lists all projects + * arguments: none +* project remove: Removes user from project + * arguments: project user +* project scrub: Deletes data associated with project + * arguments: project +* project zipfile: Exports credentials for project to a zip file + * arguments: project_id user_id [filename='nova.zip] + +Setting Quotas +-------------- +Nova utilizes a quota system at the project level to control resource consumption across available hardware resources. Current quota controls are available to limit the: + +* Number of volumes which may be created +* Total size of all volumes within a project as measured in GB +* Number of instances which may be launched +* Number of processor cores which may be allocated +* Publicly accessible IP addresses + +Use the following command to set quotas for a project +* project quota: Set or display quotas for project + * arguments: project_id [key] [value] diff --git a/doc/source/adminguide/managing.users.rst b/doc/source/adminguide/managing.users.rst new file mode 100644 index 000000000000..392142e86f8b --- /dev/null +++ b/doc/source/adminguide/managing.users.rst @@ -0,0 +1,82 @@ +Managing Users +============== + + +Users and Access Keys +--------------------- + +Access to the ec2 api is controlled by an access and secret key. The user's access key needs to be included in the request, and the request must be signed with the secret key. Upon receipt of api requests, nova will verify the signature and execute commands on behalf of the user. + +In order to begin using nova, you will need a to create a user. This can be easily accomplished using the user create or user admin commands in nova-manage. `user create` will create a regular user, whereas `user admin` will create an admin user. The syntax of the command is nova-manage user create username [access] [secret]. For example:: + + nova-manage user create john my-access-key a-super-secret-key + +If you do not specify an access or secret key, a random uuid will be created automatically. + +Credentials +----------- + +Nova can generate a handy set of credentials for a user. These credentials include a CA for bundling images and a file for setting environment variables to be used by euca2ools. If you don't need to bundle images, just the environment script is required. You can export one with the `project environment` command. The syntax of the command is nova-manage project environment project_id user_id [filename]. If you don't specify a filename, it will be exported as novarc. After generating the file, you can simply source it in bash to add the variables to your environment:: + + nova-manage project environment john_project john + . novarc + +If you do need to bundle images, you will need to get all of the credentials using `project zipfile`. Note that zipfile will give you an error message if networks haven't been created yet. Otherwise zipfile has the same syntax as environment, only the default file name is nova.zip. Example usage:: + + nova-manage project zipfile john_project john + unzip nova.zip + . novarc + +Role Based Access Control +------------------------- +Roles control the api actions that a user is allowed to perform. For example, a user cannot allocate a public ip without the `netadmin` role. It is important to remember that a users de facto permissions in a project is the intersection of user (global) roles and project (local) roles. So for john to have netadmin permissions in his project, he needs to separate roles specified. You can add roles with `role add`. The syntax is nova-manage role add user_id role [project_id]. Let's give john the netadmin role for his project:: + + nova-manage role add john netadmin + nova-manage role add john netadmin john_project + +Role-based access control (RBAC) is an approach to restricting system access to authorized users based on an individual’s role within an organization. Various employee functions require certain levels of system access in order to be successful. These functions are mapped to defined roles and individuals are categorized accordingly. Since users are not assigned permissions directly, but only acquire them through their role (or roles), management of individual user rights becomes a matter of assigning appropriate roles to the user. This simplifies common operations, such as adding a user, or changing a user's department. + +Nova’s rights management system employs the RBAC model and currently supports the following five roles: + +* **Cloud Administrator.** (admin) Users of this class enjoy complete system access. +* **IT Security.** (itsec) This role is limited to IT security personnel. It permits role holders to quarantine instances. +* **Project Manager.** (projectmanager)The default for project owners, this role affords users the ability to add other users to a project, interact with project images, and launch and terminate instances. +* **Network Administrator.** (netadmin) Users with this role are permitted to allocate and assign publicly accessible IP addresses as well as create and modify firewall rules. +* **Developer.** This is a general purpose role that is assigned to users by default. + +RBAC management is exposed through the dashboard for simplified user management. + + +User Commands +~~~~~~~~~~~~ + +Users, including admins, are created through the ``user`` commands. + +* user admin: creates a new admin and prints exports + * arguments: name [access] [secret] +* user create: creates a new user and prints exports + * arguments: name [access] [secret] +* user delete: deletes an existing user + * arguments: name +* user exports: prints access and secrets for user in export format + * arguments: name +* user list: lists all users + * arguments: none +* user modify: update a users keys & admin flag + * arguments: accesskey secretkey admin + * leave any field blank to ignore it, admin should be 'T', 'F', or blank + + +User Role Management +~~~~~~~~~~~~~~~~~~~~ + +* role add: adds role to user + * if project is specified, adds project specific role + * arguments: user, role [project] +* role has: checks to see if user has role + * if project is specified, returns True if user has + the global role and the project role + * arguments: user, role [project] +* role remove: removes role from user + * if project is specified, removes project specific role + * arguments: user, role [project] diff --git a/doc/source/adminguide/managingsecurity.rst b/doc/source/adminguide/managingsecurity.rst new file mode 100644 index 000000000000..3b11b181a3bc --- /dev/null +++ b/doc/source/adminguide/managingsecurity.rst @@ -0,0 +1,39 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Security Considerations +======================= + +.. todo:: This doc is vague and just high-level right now. Describe architecture that enables security. + +The goal of securing a cloud computing system involves both protecting the instances, data on the instances, and +ensuring users are authenticated for actions and that borders are understood by the users and the system. +Protecting the system from intrusion or attack involves authentication, network protections, and +compromise detection. + +Key Concepts +------------ + +Authentication - Each instance is authenticated with a key pair. + +Network - Instances can communicate with each other but you can configure the boundaries through firewall +configuration. + +Monitoring - Log all API commands and audit those logs. + +Encryption - Data transfer between instances is not encrypted. + diff --git a/doc/source/modules.rst b/doc/source/adminguide/monitoring.rst similarity index 79% rename from doc/source/modules.rst rename to doc/source/adminguide/monitoring.rst index 82c61f008de8..e7766a6e7415 100644 --- a/doc/source/modules.rst +++ b/doc/source/adminguide/monitoring.rst @@ -15,20 +15,13 @@ License for the specific language governing permissions and limitations under the License. -Nova Documentation -================== +Monitoring +========== -This page contains the Nova Modules documentation. +* components +* throughput +* exceptions +* hardware -Modules: --------- - -.. toctree:: - :maxdepth: 4 - - auth - compute - endpoint - fakes - nova - volume +* ganglia +* syslog diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst new file mode 100644 index 000000000000..fa0652bc87e6 --- /dev/null +++ b/doc/source/adminguide/multi.node.install.rst @@ -0,0 +1,298 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Installing Nova on Multiple Servers +=================================== + +When you move beyond evaluating the technology and into building an actual +production environemnt, you will need to know how to configure your datacenter +and how to deploy components across your clusters. This guide should help you +through that process. + +You can install multiple nodes to increase performance and availability of the OpenStack Compute installation. + +This setup is based on an Ubuntu Lucid 10.04 installation with the latest updates. Most of this works around issues that need to be resolved in the installation and configuration scripts as of October 18th 2010. It also needs to eventually be generalized, but the intent here is to get the multi-node configuration bootstrapped so folks can move forward. + + +Requirements for a multi-node installation +------------------------------------------ + +* You need a real database, compatible with SQLAlchemy (mysql, postgresql) There's not a specific reason to choose one over another, it basically depends what you know. MySQL is easier to do High Availability (HA) with, but people may already know Postgres. We should document both configurations, though. +* For a recommended HA setup, consider a MySQL master/slave replication, with as many slaves as you like, and probably a heartbeat to kick one of the slaves into being a master if it dies. +* For performance optimization, split reads and writes to the database. MySQL proxy is the easiest way to make this work if running MySQL. + + +Assumptions +^^^^^^^^^^^ + +* Networking is configured between/through the physical machines on a single subnet. +* Installation and execution are both performed by root user. + + + +Step 1 Use apt-get to get the latest code +----------------------------------------- + +1. Setup Nova PPA with https://launchpad.net/~nova-core/+archive/ppa. + +:: + + sudo apt-get install python-software-properties + sudo add-apt-repository ppa:nova-core/ppa + +2. Run update. + +:: + + sudo apt-get update + +3. Install nova-pkgs (dependencies should be automatically installed). + +:: + + sudo apt-get install python-greenlet + sudo apt-get install nova-common nova-doc python-nova nova-api nova-network nova-objectstore nova-scheduler + +It is highly likely that there will be errors when the nova services come up since they are not yet configured. Don't worry, you're only at step 1! + +Step 2 Setup configuration files (installed in /etc/nova) +--------------------------------------------------------- + +Note: CC_ADDR= + +1. These need to be defined in EACH configuration file + +:: + + --sql_connection=mysql://root:nova@$CC_ADDR/nova # location of nova sql db + --s3_host=$CC_ADDR # This is where nova is hosting the objectstore service, which + # will contain the VM images and buckets + --rabbit_host=$CC_ADDR # This is where the rabbit AMQP messaging service is hosted + --cc_host=$CC_ADDR # This is where the the nova-api service lives + --verbose # Optional but very helpful during initial setup + --ec2_url=http://$CC_ADDR:8773/services/Cloud + --network_manager=nova.network.manager.FlatManager # simple, no-vlan networking type + + +2. nova-manage specific flags + +:: + + --FAKE_subdomain=ec2 # workaround for ec2/euca api + --fixed_range= # ip network to use for VM guests, ex 192.168.2.64/26 + --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 + + +3. nova-network specific flags + +:: + + --fixed_range= # ip network to use for VM guests, ex 192.168.2.64/26 + --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 + +4. nova-api specific flags + +:: + + --FAKE_subdomain=ec2 # workaround for ec2/euca api + +5. Create a nova group + +:: + + sudo addgroup nova + +6. nova-objectstore specific flags < no specific config needed > + +Config files should be have their owner set to root:nova, and mode set to 0640, since they contain your MySQL server's root password. + +:: + + cd /etc/nova + chown -R root:nova . + +Step 3 Setup the sql db +----------------------- + +1. First you 'preseed' (using vishy's :doc:`../quickstart`). Run this as root. + +:: + + sudo apt-get install bzr git-core + sudo bash + export MYSQL_PASS=nova + + +:: + + cat < + /usr/bin/python /usr/bin/nova-manage project create + /usr/bin/python /usr/bin/nova-manage project create network + +Note: The nova-manage service assumes that the first IP address is your network (like 192.168.0.0), that the 2nd IP is your gateway (192.168.0.1), and that the broadcast is the very last IP in the range you defined (192.168.0.255). If this is not the case you will need to manually edit the sql db 'networks' table.o. + +On running this command, entries are made in the 'networks' and 'fixed_ips' table. However, one of the networks listed in the 'networks' table needs to be marked as bridge in order for the code to know that a bridge exists. We ended up doing this manually, (update query fired directly in the DB). Is there a better way to mark a network as bridged? + +Update: This has been resolved w.e.f 27/10. network is marked as bridged automatically based on the type of n/w manager selected. + +More networking details to create a network bridge for flat network +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Nova defaults to a bridge device named 'br100'. This needs to be created and somehow integrated into YOUR network. In my case, I wanted to keep things as simple as possible and have all the vm guests on the same network as the vm hosts (the compute nodes). Thus, I set the compute node's external IP address to be on the bridge and added eth0 to that bridge. To do this, edit your network interfaces config to look like the following:: + + < begin /etc/network/interfaces > + # The loopback network interface + auto lo + iface lo inet loopback + + # Networking for NOVA + auto br100 + + iface br100 inet dhcp + bridge_ports eth0 + bridge_stp off + bridge_maxwait 0 + bridge_fd 0 + < end /etc/network/interfaces > + + +Next, restart networking to apply the changes:: + + sudo /etc/init.d/networking restart + +Step 5: Create nova certs. +-------------------------- + +Generate the certs as a zip file:: + + mkdir creds + sudo /usr/bin/python /usr/bin/nova-manage project zip admin admin creds/nova.zip + +you can get the rc file more easily with:: + + sudo /usr/bin/python /usr/bin/nova-manage project env admin admin creds/novarc + +unzip them in your home directory, and add them to your environment:: + + unzip creds/nova.zip + echo ". creds/novarc" >> ~/.bashrc + ~/.bashrc + + +Step 6 Restart all relevant services +------------------------------------ + +Restart Libvirt:: + + sudo /etc/init.d/libvirt-bin restart + +Restart relevant nova services:: + + sudo /etc/init.d/nova-compute restart + sudo /etc/init.d/nova-volume restart + + +.. todo:: do we still need the content below? + +Bare-metal Provisioning +----------------------- + +To install the base operating system you can use PXE booting. + +Types of Hosts +-------------- + +A single machine in your cluster can act as one or more of the following types +of host: + +Nova Services + +* Network +* Compute +* Volume +* API +* Objectstore + +Other supporting services + +* Message Queue +* Database (optional) +* Authentication database (optional) + +Initial Setup +------------- + +* Networking +* Cloudadmin User Creation + +Deployment Technologies +----------------------- + +Once you have machines with a base operating system installation, you can deploy +code and configuration with your favorite tools to specify which machines in +your cluster have which roles: + +* Puppet +* Chef diff --git a/doc/source/adminguide/network.flat.rst b/doc/source/adminguide/network.flat.rst new file mode 100644 index 000000000000..1b8661a40382 --- /dev/null +++ b/doc/source/adminguide/network.flat.rst @@ -0,0 +1,60 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +Flat Network Mode (Original and Flat) +===================================== + +Flat network mode removes most of the complexity of VLAN mode by simply +bridging all instance interfaces onto a single network. + +There are two variations of flat mode that differ mostly in how IP addresses +are given to instances. + + +Original Flat Mode +------------------ +IP addresses for VM instances are grabbed from a subnet specified by the network administrator, and injected into the image on launch. All instances of the system are attached to the same Linux networking bridge, configured manually by the network administrator both on the network controller hosting the network and on the computer controllers hosting the instances. To recap: + +* Each compute host creates a single bridge for all instances to use to attach to the external network. +* The networking configuration is injected into the instance before it is booted or it is obtained by a guest agent installed in the instance. + +Note that the configuration injection currently only works on linux-style systems that keep networking +configuration in /etc/network/interfaces. + + +Flat DHCP Mode +-------------- +IP addresses for VM instances are grabbed from a subnet specified by the network administrator. Similar to the flat network, a single Linux networking bridge is created and configured manually by the network administrator and used for all instances. A DHCP server is started to pass out IP addresses to VM instances from the specified subnet. To recap: + +* Like flat mode, all instances are attached to a single bridge on the compute node. +* In addition a DHCP server is running to configure instances. + +Implementation +-------------- + +The network nodes do not act as a default gateway in flat mode. Instances +are given public IP addresses. + +Compute nodes have iptables/ebtables entries created per project and +instance to protect against IP/MAC address spoofing and ARP poisoning. + + +Examples +-------- + +.. todo:: add flat network mode configuration examples diff --git a/doc/source/adminguide/network.vlan.rst b/doc/source/adminguide/network.vlan.rst new file mode 100644 index 000000000000..a7cccc098762 --- /dev/null +++ b/doc/source/adminguide/network.vlan.rst @@ -0,0 +1,179 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +VLAN Network Mode +================= +VLAN Network Mode is the default mode for Nova. It provides a private network +segment for each project's instances that can be accessed via a dedicated +VPN connection from the Internet. + +In this mode, each project gets its own VLAN, Linux networking bridge, and subnet. The subnets are specified by the network administrator, and are assigned dynamically to a project when required. A DHCP Server is started for each VLAN to pass out IP addresses to VM instances from the subnet assigned to the project. All instances belonging to one project are bridged into the same VLAN for that project. The Linux networking bridges and VLANs are created by Nova when required, described in more detail in Nova VLAN Network Management Implementation. + +.. + (this text revised above) + Because the flat network and flat DhCP network are simple to understand and yet do not scale well enough for real-world cloud systems, this section focuses on the VLAN network implementation by the VLAN Network Manager. + + + In the VLAN network mode, all the VM instances of a project are connected together in a VLAN with the specified private subnet. Each running VM instance is assigned an IP address within the given private subnet. + +.. todo:: Insert Figure 2 from "An OpenStack Network Overview" contributed by Citrix + +While network traffic between VM instances belonging to the same VLAN is always open, Nova can enforce isolation of network traffic between different projects by enforcing one VLAN per project. + +In addition, the network administrator can specify a pool of public IP addresses that users may allocate and then assign to VMs, either at boot or dynamically at run-time. This capability is similar to Amazon's 'elastic IPs'. A public IP address may be associated with a running instances, allowing the VM instance to be accessed from the public network. The public IP addresses are accessible from the network host and NATed to the private IP address of the project. + +.. todo:: Describe how a public IP address could be associated with a project (a VLAN) + +This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code named :ref:`cloudpipe `) needs to be created. Nova generates a certificate and key for the user to access the vpn and starts the vpn automatically. More information on cloudpipe can be found :ref:`here `. + +The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) + +.. image:: /images/cloudpipe.png + :width: 100% + +Goals +----- + +* each project is in a protected network segment + + * RFC-1918 IP space + * public IP via NAT + * no default inbound Internet access without public NAT + * limited (project-admin controllable) outbound Internet access + * limited (project-admin controllable) access to other project segments + * all connectivity to instance and cloud API is via VPN into the project segment + +* common DMZ segment for support services (only visible from project segment) + + * metadata + * dashboard + + +Limitations +----------- + +* Projects / cluster limited to available VLANs in switching infrastructure +* Requires VPN for access to project segment + + +Implementation +-------------- +Currently Nova segregates project VLANs using 802.1q VLAN tagging in the +switching layer. Compute hosts create VLAN-specific interfaces and bridges +as required. + +The network nodes act as default gateway for project networks and contain +all of the routing and firewall rules implementing security groups. The +network node also handles DHCP to provide instance IPs for each project. + +VPN access is provided by running a small instance called CloudPipe +on the IP immediately following the gateway IP for each project. The +network node maps a dedicated public IP/port to the CloudPipe instance. + +Compute nodes have per-VLAN interfaces and bridges created as required. +These do NOT have IP addresses in the host to protect host access. +Compute nodes have iptables/ebtables entries created per project and +instance to protect against IP/MAC address spoofing and ARP poisoning. + +The network assignment to a project, and IP address assignment to a VM instance, are triggered when a user starts to run a VM instance. When running a VM instance, a user needs to specify a project for the instances, and the security groups (described in Security Groups) when the instance wants to join. If this is the first instance to be created for the project, then Nova (the cloud controller) needs to find a network controller to be the network host for the project; it then sets up a private network by finding an unused VLAN id, an unused subnet, and then the controller assigns them to the project, it also assigns a name to the project's Linux bridge, and allocating a private IP within the project's subnet for the new instance. + +If the instance the user wants to start is not the project's first, a subnet and a VLAN must have already been assigned to the project; therefore the system needs only to find an available IP address within the subnet and assign it to the new starting instance. If there is no private IP available within the subnet, an exception will be raised to the cloud controller, and the VM creation cannot proceed. + +.. todo:: insert the name of the Linux bridge, is it always named bridge? + +External Infrastructure +----------------------- + +Nova assumes the following is available: + +* DNS +* NTP +* Internet connectivity + + +Example +------- + +This example network configuration demonstrates most of the capabilities +of VLAN Mode. It splits administrative access to the nodes onto a dedicated +management network and uses dedicated network nodes to handle all +routing and gateway functions. + +It uses a 10GB network for instance traffic and a 1GB network for management. + + +Hardware +~~~~~~~~ + +* All nodes have a minimum of two NICs for management and production. + + * management is 1GB + * production is 10GB + * add additional NICs for bonding or HA/performance + +* network nodes should have an additional NIC dedicated to public Internet traffic +* switch needs to support enough simultaneous VLANs for number of projects +* production network configured as 802.1q trunk on switch + + +Operation +~~~~~~~~~ + +The network node controls the project network configuration: + +* assigns each project a VLAN and private IP range +* starts dnsmasq on project VLAN to serve private IP range +* configures iptables on network node for default project access +* launches CloudPipe instance and configures iptables access + +When starting an instance the network node: + +* sets up a VLAN interface and bridge on each host as required when an + instance is started on that host +* assigns private IP to instance +* generates MAC address for instance +* update dnsmasq with IP/MAC for instance + +When starting an instance the compute node: + +* sets up a VLAN interface and bridge on each host as required when an + instance is started on that host + + +Setup +~~~~~ + +* Assign VLANs in the switch: + + * public Internet segment + * production network + * management network + * cluster DMZ + +* Assign a contiguous range of VLANs to Nova for project use. +* Configure management NIC ports as management VLAN access ports. +* Configure management VLAN with Internet access as required +* Configure production NIC ports as 802.1q trunk ports. +* Configure Nova (need to add specifics here) + + * public IPs + * instance IPs + * project network size + * DMZ network + +.. todo:: need specific Nova configuration added diff --git a/doc/source/adminguide/nova.manage.rst b/doc/source/adminguide/nova.manage.rst new file mode 100644 index 000000000000..89fb396691e6 --- /dev/null +++ b/doc/source/adminguide/nova.manage.rst @@ -0,0 +1,116 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +The nova-manage command +======================= + +Introduction +~~~~~~~~~~~~ + +The nova-manage command is used to perform many essential functions for +administration and ongoing maintenance of nova, such as user creation, +vpn management, and much more. + +The standard pattern for executing a nova-manage command is: + +``nova-manage []`` + +For example, to obtain a list of all projects: + +``nova-manage project list`` + +You can run without arguments to see a list of available command categories: + +``nova-manage`` + +You can run with a category argument to see a list of all commands in that +category: + +``nova-manage user`` + + +Nova Shell +~~~~~~~~~~ + +* shell bpython + * start a new bpython shell +* shell ipython + * start a new ipython shell +* shell python + * start a new python shell +* shell run + * ??? +* shell script: Runs the script from the specifed path with flags set properly. + * arguments: path + + +Concept: Flags +-------------- + +python-gflags + + +Concept: Plugins +---------------- + +* Managers/Drivers: utils.import_object from string flag +* virt/connections: conditional loading from string flag +* db: LazyPluggable via string flag +* auth_manager: utils.import_class based on string flag +* Volumes: moving to pluggable driver instead of manager +* Network: pluggable managers +* Compute: same driver used, but pluggable at connection + + +Concept: IPC/RPC +---------------- + +Rabbit! + + +Concept: Fakes +-------------- + +* auth +* ldap + + +Concept: Scheduler +------------------ + +* simple +* random + + +Concept: Security Groups +------------------------ + +Security groups + + +Concept: Certificate Authority +------------------------------ + +Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns <../cloudpipe>` and decrypting bundled images. + + +Concept: Images +--------------- + +* launching +* bundling diff --git a/doc/source/adminguide/single.node.install.rst b/doc/source/adminguide/single.node.install.rst new file mode 100644 index 000000000000..27597962aae6 --- /dev/null +++ b/doc/source/adminguide/single.node.install.rst @@ -0,0 +1,344 @@ +Installing Nova on a Single Host +================================ + +Nova can be run on a single machine, and it is recommended that new users practice managing this type of installation before graduating to multi node systems. + +The fastest way to get a test cloud running is through our :doc:`../quickstart`. But for more detail on installing the system read this doc. + + +Step 1 and 2: Get the latest Nova code system software +------------------------------------------------------ + +Depending on your system, the mehod for accomplishing this varies + +.. toctree:: + :maxdepth: 1 + + distros/ubuntu.10.04 + distros/ubuntu.10.10 + distros/others + + +Step 3: Build and install Nova services +--------------------------------------- + +Switch to the base nova source directory. + +Then type or copy/paste in the following line to compile the Python code for OpenStack Compute. + +:: + + sudo python setup.py build + sudo python setup.py install + + +When the installation is complete, you'll see the following lines: + +:: + + Installing nova-network script to /usr/local/bin + Installing nova-volume script to /usr/local/bin + Installing nova-objectstore script to /usr/local/bin + Installing nova-manage script to /usr/local/bin + Installing nova-scheduler script to /usr/local/bin + Installing nova-dhcpbridge script to /usr/local/bin + Installing nova-compute script to /usr/local/bin + Installing nova-instancemonitor script to /usr/local/bin + Installing nova-api script to /usr/local/bin + Installing nova-import-canonical-imagestore script to /usr/local/bin + + Installed /usr/local/lib/python2.6/dist-packages/nova-2010.1-py2.6.egg + Processing dependencies for nova==2010.1 + Finished processing dependencies for nova==2010.1 + + +Step 4: Create a Nova administrator +----------------------------------- +Type or copy/paste in the following line to create a user named "anne.":: + + sudo nova-manage user admin anne + +You see an access key and a secret key export, such as these made-up ones::: + + export EC2_ACCESS_KEY=4e6498a2-blah-blah-blah-17d1333t97fd + export EC2_SECRET_KEY=0a520304-blah-blah-blah-340sp34k05bbe9a7 + + +Step 5: Create a project with the user you created +-------------------------------------------------- +Type or copy/paste in the following line to create a project named IRT (for Ice Road Truckers, of course) with the newly-created user named anne. + +:: + + sudo nova-manage project create IRT anne + +:: + + Generating RSA private key, 1024 bit long modulus + .....++++++ + ..++++++ + e is 65537 (0x10001) + Using configuration from ./openssl.cnf + Check that the request matches the signature + Signature ok + The Subject's Distinguished Name is as follows + countryName :PRINTABLE:'US' + stateOrProvinceName :PRINTABLE:'California' + localityName :PRINTABLE:'MountainView' + organizationName :PRINTABLE:'AnsoLabs' + organizationalUnitName:PRINTABLE:'NovaDev' + commonName :PRINTABLE:'anne-2010-10-12T21:12:35Z' + Certificate is to be certified until Oct 12 21:12:35 2011 GMT (365 days) + + Write out database with 1 new entries + Data Base Updated + + +Step 6: Unzip the nova.zip +-------------------------- + +You should have a nova.zip file in your current working directory. Unzip it with this command: + +:: + + unzip nova.zip + + +You'll see these files extract. + +:: + + Archive: nova.zip + extracting: novarc + extracting: pk.pem + extracting: cert.pem + extracting: nova-vpn.conf + extracting: cacert.pem + + +Step 7: Source the rc file +-------------------------- +Type or copy/paste the following to source the novarc file in your current working directory. + +:: + + . novarc + + +Step 8: Pat yourself on the back :) +----------------------------------- +Congratulations, your cloud is up and running, you’ve created an admin user, retrieved the user's credentials and put them in your environment. + +Now you need an image. + + +Step 9: Get an image +-------------------- +To make things easier, we've provided a small image on the Rackspace CDN. Use this command to get it on your server. + +:: + + wget http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz + + +:: + + --2010-10-12 21:40:55-- http://c2477062.cdn.cloudfiles.rackspacecloud.com/images.tgz + Resolving cblah2.cdn.cloudfiles.rackspacecloud.com... 208.111.196.6, 208.111.196.7 + Connecting to cblah2.cdn.cloudfiles.rackspacecloud.com|208.111.196.6|:80... connected. + HTTP request sent, awaiting response... 200 OK + Length: 58520278 (56M) [appication/x-gzip] + Saving to: `images.tgz' + + 100%[======================================>] 58,520,278 14.1M/s in 3.9s + + 2010-10-12 21:40:59 (14.1 MB/s) - `images.tgz' saved [58520278/58520278] + + + +Step 10: Decompress the image file +---------------------------------- +Use this command to extract the image files::: + + tar xvzf images.tgz + +You get a directory listing like so::: + + images + |-- aki-lucid + | |-- image + | `-- info.json + |-- ami-tiny + | |-- image + | `-- info.json + `-- ari-lucid + |-- image + `-- info.json + +Step 11: Send commands to upload sample image to the cloud +---------------------------------------------------------- + +Type or copy/paste the following commands to create a manifest for the kernel.:: + + euca-bundle-image -i images/aki-lucid/image -p kernel --kernel true + +You should see this in response::: + + Checking image + Tarring image + Encrypting image + Splitting image... + Part: kernel.part.0 + Generating manifest /tmp/kernel.manifest.xml + +Type or copy/paste the following commands to create a manifest for the ramdisk.:: + + euca-bundle-image -i images/ari-lucid/image -p ramdisk --ramdisk true + +You should see this in response::: + + Checking image + Tarring image + Encrypting image + Splitting image... + Part: ramdisk.part.0 + Generating manifest /tmp/ramdisk.manifest.xml + +Type or copy/paste the following commands to upload the kernel bundle.:: + + euca-upload-bundle -m /tmp/kernel.manifest.xml -b mybucket + +You should see this in response::: + + Checking bucket: mybucket + Creating bucket: mybucket + Uploading manifest file + Uploading part: kernel.part.0 + Uploaded image as mybucket/kernel.manifest.xml + +Type or copy/paste the following commands to upload the ramdisk bundle.:: + + euca-upload-bundle -m /tmp/ramdisk.manifest.xml -b mybucket + +You should see this in response::: + + Checking bucket: mybucket + Uploading manifest file + Uploading part: ramdisk.part.0 + Uploaded image as mybucket/ramdisk.manifest.xml + +Type or copy/paste the following commands to register the kernel and get its ID.:: + + euca-register mybucket/kernel.manifest.xml + +You should see this in response::: + + IMAGE ami-fcbj2non + +Type or copy/paste the following commands to register the ramdisk and get its ID.:: + + euca-register mybucket/ramdisk.manifest.xml + +You should see this in response::: + + IMAGE ami-orukptrc + +Type or copy/paste the following commands to create a manifest for the machine image associated with the ramdisk and kernel IDs that you got from the previous commands.:: + + euca-bundle-image -i images/ami-tiny/image -p machine --kernel ami-fcbj2non --ramdisk ami-orukptrc + +You should see this in response::: + + Checking image + Tarring image + Encrypting image + Splitting image... + Part: machine.part.0 + Part: machine.part.1 + Part: machine.part.2 + Part: machine.part.3 + Part: machine.part.4 + Generating manifest /tmp/machine.manifest.xml + +Type or copy/paste the following commands to upload the machine image bundle.:: + + euca-upload-bundle -m /tmp/machine.manifest.xml -b mybucket + +You should see this in response::: + + Checking bucket: mybucket + Uploading manifest file + Uploading part: machine.part.0 + Uploading part: machine.part.1 + Uploading part: machine.part.2 + Uploading part: machine.part.3 + Uploading part: machine.part.4 + Uploaded image as mybucket/machine.manifest.xml + +Type or copy/paste the following commands to register the machine image and get its ID.:: + + euca-register mybucket/machine.manifest.xml + +You should see this in response::: + + IMAGE ami-g06qbntt + +Type or copy/paste the following commands to register a SSH keypair for use in starting and accessing the instances.:: + + euca-add-keypair mykey > mykey.priv + chmod 600 mykey.priv + +Type or copy/paste the following commands to run an instance using the keypair and IDs that we previously created.:: + + euca-run-instances ami-g06qbntt --kernel ami-fcbj2non --ramdisk ami-orukptrc -k mykey + +You should see this in response::: + + RESERVATION r-0at28z12 IRT + INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 scheduling mykey (IRT, None) m1.small 2010-10-18 19:02:10.443599 + +Type or copy/paste the following commands to watch as the scheduler launches, and completes booting your instance.:: + + euca-describe-instances + +You should see this in response::: + + RESERVATION r-0at28z12 IRT + INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 launching mykey (IRT, cloud02) m1.small 2010-10-18 19:02:10.443599 + +Type or copy/paste the following commands to see when loading is completed and the instance is running.:: + + euca-describe-instances + +You should see this in response::: + + RESERVATION r-0at28z12 IRT + INSTANCE i-1b0bh8n ami-g06qbntt 10.0.0.3 10.0.0.3 running mykey (IRT, cloud02) 0 m1.small 2010-10-18 19:02:10.443599 + +Type or copy/paste the following commands to check that the virtual machine is running.:: + + virsh list + +You should see this in response::: + + Id Name State + ---------------------------------- + 1 2842445831 running + +Type or copy/paste the following commands to ssh to the instance using your private key.:: + + ssh -i mykey.priv root@10.0.0.3 + + +Troubleshooting Installation +---------------------------- + +If you see an "error loading the config file './openssl.cnf'" it means you can copy the openssl.cnf file to the location where Nova expects it and reboot, then try the command again. + +:: + + cp /etc/ssl/openssl.cnf ~ + sudo reboot + + + diff --git a/doc/source/architecture.rst b/doc/source/architecture.rst deleted file mode 100644 index 11813d2c8c3a..000000000000 --- a/doc/source/architecture.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -nova System Architecture -======================== - -Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received. - -In order to achieve shared-nothing with multiple copies of the same component (especially when the component is an API server that needs to reply with state information in a timely fashion), we need to keep all of our system state in a distributed data system. Updates to system state are written into this system, using atomic transactions when necessary. Requests for state are read out of this system. In limited cases, these read calls are memoized within controllers for short periods of time. (Such a limited case would be, for instance, the current list of system users.) - - -Components ----------- - -Below you will find a helpful explanation. - -:: - - [ User Manager ] ---- ( LDAP ) - | - | / [ Storage ] - ( ATAoE ) - [ API server ] -> [ Cloud ] < AMQP > - | \ [ Nodes ] - ( libvirt/kvm ) - < HTTP > - | - [ S3 ] - - -* API: receives http requests from boto, converts commands to/from API format, and sending requests to cloud controller -* Cloud Controller: global state of system, talks to ldap, s3, and node/storage workers through a queue -* Nodes: worker that spawns instances -* S3: tornado based http/s3 server -* User Manager: create/manage users, which are stored in ldap -* Network Controller: allocate and deallocate IPs and VLANs diff --git a/doc/source/cloud101.rst b/doc/source/cloud101.rst new file mode 100644 index 000000000000..87db5af1eb77 --- /dev/null +++ b/doc/source/cloud101.rst @@ -0,0 +1,85 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Cloud Computing 101 +=================== + +Originally the term cloud came from a diagram that contained a cloud-like shape to contain the +services that afforded computing power that was harnessed to get work done. Much like the electrical +power we receive each day, cloud computing is a model for enabling access to a shared collection of +computing resources - networks for transfer, servers for storage, and applications or services for +completing work. + +Why Cloud? +---------- +Like humans supposedly only use 10% of their brain power, many of the computers in place in data +centers today are underutilized in computing power and networking bandwidth. People also may need a large +amount of computing capacity to complete a computation for example, but don't need the computing power +once the computation is done. You want cloud computing when you want a service that's available +on-demand with the flexibility to bring it up or down through automation or with little intervention. + +Attributes of a Cloud +--------------------- +On-demand self-service - A cloud should enable self-service, so that users can provision servers and networks with little +human intervention. + +Network access - Any computing capabilities are available over the network and you can use many different +devices through standardized mechanisms. + +Resource pooling - Clouds can serve multiple consumers according to demand. + +Elasticity - Provisioning is rapid and scales out or in based on need. + +Metered or measured service - Just like utilities that are paid for by the hour, clouds should optimize +resource use and control it for the level of service or type of servers such as storage or processing. + +Types of Cloud Services +----------------------- + +Cloud computing offers different service models depending on the capabilities a consumer may require. +The US-based National Institute of Standards and Technology offers definitions for cloud computing +and the service models that are emerging. + +SaaS - Software as a Service +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Provides the consumer the ability to use the software in a cloud environment, such as web-based email for example. + +PaaS - Platform as a Service +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Provides the consumer the ability to deploy applications through a programming language or tools supported +by the cloud platform provider. An example of platform as a service is an Eclipse/Java programming +platform provided with no downloads required. + +IaaS - Infrastructure as a Service +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Provides infrastructure such as computer instances, network connections, and storage so that people +can run any software or operating system. + +.. todo:: Use definitions from http://csrc.nist.gov/groups/SNS/cloud-computing/ and attribute NIST + +Types of Cloud Deployments +-------------------------- +.. todo:: describe public/private/hybrid/etc + + +Work in the Clouds +------------------ + +.. todo:: What people have done/sample projects diff --git a/doc/source/community.rst b/doc/source/community.rst new file mode 100644 index 000000000000..bfb93414ce1e --- /dev/null +++ b/doc/source/community.rst @@ -0,0 +1,84 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Getting Involved +================ + +The Nova community is a very friendly group and there are places online to join in with the +community. Feel free to ask questions. This document points you to some of the places where you can +communicate with people. + +How to Join the OpenStack Community +----------------------------------- + +Our community welcomes all people interested in open source cloud computing, and there are no formal +membership requirements. The best way to join the community is to talk with others online or at a meetup +and offer contributions through Launchpad, the wiki, or blogs. We welcome all types of contributions, +from blueprint designs to documentation to testing to deployment scripts. + +Contributing Code +----------------- + +To contribute code, sign up for a Launchpad account and sign a contributor license agreement, +available on the `OpenStack Wiki `_. Once the CLA is signed you +can contribute code through the Bazaar version control system which is related to your Launchpad account. + +#openstack on Freenode IRC Network +---------------------------------- + +There is a very active chat channel at ``_. This +is usually the best place to ask questions and find your way around. IRC stands for Internet Relay +Chat and it is a way to chat online in real time. You can also ask a question and come back to the +log files to read the answer later. Logs for the #openstack IRC channel are stored at +``_. + +OpenStack Wiki +-------------- + +The wiki is a living source of knowledge. It is edited by the community, and +has collections of links and other sources of information. Typically the pages are a good place +to write drafts for specs or documentation, describe a blueprint, or collaborate with others. + +`OpenStack Wiki `_ + +Nova on Launchpad +----------------- + +Launchpad is a code hosting service that hosts the Nova source code. From +Launchpad you can report bugs, ask questions, and register blueprints (feature requests). + +* `Learn about how to use bzr with launchpad `_ +* `Launchpad Nova Page `_ + +OpenStack Blog +-------------- + +The OpenStack blog includes a weekly newsletter that aggregates OpenStack news +from around the internet, as well as providing inside information on upcoming +events and posts from OpenStack contributors. + +`OpenStack Blog `_ + +See also: `Planet OpenStack `_, aggregating blogs +about OpenStack from around the internet into a single feed. If you'd like to contribute to this blog +aggregation with your blog posts, there are instructions for `adding your blog `_. + +Twitter +------- + +Because all the cool kids do it: `@openstack `_. Also follow the +`#openstack `_ tag for relevant tweets. diff --git a/doc/source/conf.py b/doc/source/conf.py index e137e728acc3..ef447ca81732 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,14 +16,22 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append([os.path.abspath('../nova'), os.path.abspath('..'), os.path.abspath('../bin')]) - +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig'] + +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'ext.nova_todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig','sphinx.ext.graphviz'] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable +if not os.getenv('SPHINX_DEBUG'): + extensions += ['ext.nova_autodoc'] + todo_include_todos = True # Add any paths that contain templates here, relative to this directory. @@ -99,7 +107,8 @@ modindex_common_prefix = ['nova.'] # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'default' +html_theme_path = ["."] +html_theme = '_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/source/devref/api.rst b/doc/source/devref/api.rst new file mode 100644 index 000000000000..14181529a976 --- /dev/null +++ b/doc/source/devref/api.rst @@ -0,0 +1,296 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +API Endpoint +============ + +Nova has a system for managing multiple APIs on different subdomains. +Currently there is support for the OpenStack API, as well as the Amazon EC2 +API. + +Common Components +----------------- + +The :mod:`nova.api` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.api.cloud` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.cloud + :noindex: + :members: + :undoc-members: + :show-inheritance: + +OpenStack API +------------- + +The :mod:`openstack` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`auth` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.auth + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`backup_schedules` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.backup_schedules + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`faults` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.faults + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`flavors` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.flavors + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.images + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`ratelimiting` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.ratelimiting + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`servers` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.servers + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`sharedipgroups` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: nova.api.openstack.sharedipgroups + :noindex: + :members: + :undoc-members: + :show-inheritance: + +EC2 API +------- + +The :mod:`nova.api.ec2` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2 + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`admin` Module +~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.admin + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`apirequest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.apirequest + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud` Module +~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.cloud + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`images` Module +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.images + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`metadatarequesthandler` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.ec2.metadatarequesthandler + :noindex: + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`api_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api_integration` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api_integration + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`cloud_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.cloud_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.fakes` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.fakes + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`api.test_wsgi` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.test_wsgi + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_api` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_api + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_auth` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_auth + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_faults` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_faults + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_flavors` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_flavors + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_images` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_images + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_ratelimiting` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_ratelimiting + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_servers` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_servers + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`test_sharedipgroups` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.api.openstack.test_sharedipgroups + :noindex: + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/devref/architecture.rst b/doc/source/devref/architecture.rst new file mode 100644 index 000000000000..1e23e13617c8 --- /dev/null +++ b/doc/source/devref/architecture.rst @@ -0,0 +1,52 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova System Architecture +======================== + +Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received. + +Nova recently moved to using a sql-based central database that is shared by all components in the system. The amount and depth of the data fits into a sql database quite well. For small deployments this seems like an optimal solution. For larger deployments, and especially if security is a concern, nova will be moving towards multiple data stores with some kind of aggregation system. + +Components +---------- + +Below you will find a helpful explanation of the different components. + +:: + + /- ( LDAP ) + [ Auth Manager ] --- + | \- ( DB ) + | + | [ scheduler ] - [ volume ] - ( ATAoE/iSCSI ) + | / + [ Web Dashboard ] -> [ api ] -- < AMQP > ------ [ network ] - ( Flat/Vlan ) + | \ + < HTTP > [ scheduler ] - [ compute ] - ( libvirt/xen ) + | | + [ objectstore ] < - retrieves images + +* DB: sql database for data storage. Used by all components (LINKS NOT SHOWN) +* Web Dashboard: potential external component that talks to the api +* api: component that receives http requests, converts commands and communicates with other components via the queue or http (in the case of objectstore) +* Auth Manager: component responsible for users/projects/and roles. Can backend to DB or LDAP. This is not a separate binary, but rather a python class that is used by most components in the system. +* objectstore: twisted http server that replicates s3 api and allows storage and retrieval of images +* scheduler: decides which host gets each vm and volume +* volume: manages dynamically attachable block devices. +* network: manages ip forwarding, bridges, and vlans +* compute: manages communication with hypervisor and virtual machines. diff --git a/doc/source/auth.rst b/doc/source/devref/auth.rst similarity index 71% rename from doc/source/auth.rst rename to doc/source/devref/auth.rst index 3fcb309cd5b5..c3af3f945e45 100644 --- a/doc/source/auth.rst +++ b/doc/source/devref/auth.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,20 +15,113 @@ License for the specific language governing permissions and limitations under the License. -Auth Documentation -================== +.. _auth: + +Authentication and Authorization +================================ + +The :mod:`nova.quota` Module +---------------------------- + +.. automodule:: nova.quota + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.auth.signer` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.signer + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Auth Manager +------------ + +The :mod:`nova.auth.manager` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.manager + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.auth.ldapdriver` Driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.ldapdriver + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.auth.dbdriver` Driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.auth.dbdriver + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Tests +----- + + +The :mod:`auth_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.auth_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`access_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.access_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`quota_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.quota_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Legacy Docs +----------- Nova provides RBAC (Role-based access control) of the AWS-type APIs. We define the following roles: Roles-Based Access Control of AWS-style APIs using SAML Assertions “Achieving FIPS 199 Moderate certification of a hybrid cloud environment using CloudAudit and declarative C.I.A. classifications” + Introduction --------------- +------------ We will investigate one method for integrating an AWS-style API with US eAuthentication-compatible federated authentication systems, to achieve access controls and limits based on traditional operational roles. Additionally, we will look at how combining this approach, with an implementation of the CloudAudit APIs, will allow us to achieve a certification under FIPS 199 Moderate classification for a hybrid cloud environment. + Relationship of US eAuth to RBAC -------------------------------- @@ -39,65 +132,71 @@ Typical implementations of US eAuth authentication systems are structured as fol [ SUN Identity Manager or other SAML Policy Controller ] --> maps URLs to groups… [ Apache Policy Agent in front of eAuth-secured Web Application ] - + In more ideal implementations, the remainder of the application-specific account information is stored either in extended schema on the LDAP server itself, via the use of a translucent LDAP proxy, or in an independent datastore keyed off of the UID provided via SAML assertion. -Basic AWS API call structure ----------------------------- +.. _auth_roles: -AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: -* System User -* System Administrator +Roles +----- + +AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles: + +* Base User +* System Administrator/Developer (currently have the same permissions) * Network Administrator * Project Manager -* Cloud Administrator -* (IT-Sec?) +* Cloud Administrator/IT-Security (currently have the same permissions) -There is an additional, conceptual end-user that may or may not have API access: +There is an additional, conceptual end-user that may or may not have API access: -* (EXTERNAL) End-user / Third-party User +* (EXTERNAL) End-user / Third-party User -Basic operations are available to any System User: +Basic operations are available to any : -* Launch Instance -* Terminate Instance (their own) -* Create keypair -* Delete keypair -* Create, Upload, Delete: Buckets and Keys (Object Store) – their own -* Create, Attach, Delete Volume (Block Store) – their own +* Describe Instances +* Describe Images +* Describe Volumes +* Describe Keypairs +* Create Keypair +* Delete Keypair +* Create, Upload, Delete: Buckets and Keys (Object Store) -System Administrators: +System Administrators/Developers/Project Manager: +* Create, Attach, Delete Volume (Block Store) +* Launch, Reboot, Terminate Instance * Register/Unregister Machine Image (project-wide) -* Change Machine Image properties (public / private) * Request / Review CloudAudit Scans -Network Administrator: - -* Change Firewall Rules, define Security Groups -* Allocate, Associate, Deassociate Public IP addresses - Project Manager: -* Launch and Terminate Instances (project-wide) -* CRUD of Object and Block store (project-wide) +* Add and remove other users (currently no api) +* Set roles (currently no api) -Cloud Administrator: +Network Administrator: + +* Change Machine Image properties (public / private) +* Change Firewall Rules, define Security Groups +* Allocate, Associate, Deassociate Public IP addresses + +Cloud Administrator/IT-Security: + +* All permissions -* Register / Unregister Kernel and Ramdisk Images -* Register / Unregister Machine Image (any) Enhancements ------------ -* SAML Token passing +* SAML Token passing * REST interfaces * SOAP interfaces Wrapping the SAML token into the API calls. Then store the UID (fetched via backchannel) into the instance metadata, providing end-to-end auditability of ownership and responsibility, without PII. + CloudAudit APIs --------------- @@ -108,8 +207,9 @@ CloudAudit APIs CloudAudit queries may spawn long-running processes (similar to launching instances, etc.) They need to return a ReservationId in the same fashion, which can be returned in further queries for updates. RBAC of CloudAudit API calls is critical, since detailed system information is a system vulnerability. + Type declarations ---------------------- +----------------- * Data declarations – Volumes and Objects * System declarations – Instances @@ -119,40 +219,44 @@ Existing API calls to launch instances specific a single, combined “type” fl These additional parameters would also apply to creation of block storage volumes (along with the existing parameter of ‘size’), and creation of object storage ‘buckets’. (C.I.A. classifications on a bucket would be inherited by the keys within this bucket.) + Request Brokering ----------------- - * Cloud Interop - * IMF Registration / PubSub - * Digital C&A +* Cloud Interop +* IMF Registration / PubSub +* Digital C&A Establishing declarative semantics for individual API calls will allow the cloud environment to seamlessly proxy these API calls to external, third-party vendors – when the requested CIA levels match. See related work within the Infrastructure 2.0 working group for more information on how the IMF Metadata specification could be utilized to manage registration of these vendors and their C&A credentials. + Dirty Cloud – Hybrid Data Centers --------------------------------- * CloudAudit bridge interfaces * Anything in the ARP table -A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. +A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network. This interconnect is typically a bridged VPN connection. Any machines that can be bridged into a hybrid environment in this fashion (at Layer 2) must implement a minimum version of the CloudAudit spec, such that they can be queried to provide a complete picture of the IT-sec runtime environment. Network discovery protocols (ARP, CDP) can be applied in this case, and existing protocols (SNMP location data, DNS LOC records) overloaded to provide CloudAudit information. + The Details ----------- - * Preliminary Roles Definitions - * Categorization of available API calls - * SAML assertion vocabulary +* Preliminary Roles Definitions +* Categorization of available API calls +* SAML assertion vocabulary + System limits ------------- -The following limits need to be defined and enforced: +The following limits need to be defined and enforced: * Total number of instances allowed (user / project) * Total number of instances, per instance type (user / project) @@ -165,43 +269,8 @@ The following limits need to be defined and enforced: Further Challenges ------------------ - * Prioritization of users / jobs in shared computing environments - * Incident response planning - * Limit launch of instances to specific security groups based on AMI - * Store AMIs in LDAP for added property control - - - -The :mod:`signer` Module ------------------------- - -.. automodule:: nova.auth.signer - :members: - :undoc-members: - :show-inheritance: - -The :mod:`users` Module ------------------------ - -.. automodule:: nova.auth.users - :members: - :undoc-members: - :show-inheritance: - -The :mod:`users_unittest` Module --------------------------------- - -.. automodule:: nova.tests.users_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`access_unittest` Module ---------------------------------- - -.. automodule:: nova.tests.access_unittest - :members: - :undoc-members: - :show-inheritance: - +* Prioritization of users / jobs in shared computing environments +* Incident response planning +* Limit launch of instances to specific security groups based on AMI +* Store AMIs in LDAP for added property control diff --git a/doc/source/devref/cloudpipe.rst b/doc/source/devref/cloudpipe.rst new file mode 100644 index 000000000000..31bd85e8178f --- /dev/null +++ b/doc/source/devref/cloudpipe.rst @@ -0,0 +1,95 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +.. _cloudpipe: + +Cloudpipe -- Per Project Vpns +============================= + +Cloudpipe is a method for connecting end users to their project insnances in vlan mode. + + +Overview +-------- + +The support code for cloudpipe implements admin commands (via nova-manage) to automatically create a vm for a project that allows users to vpn into the private network of their project. Access to this vpn is provided through a public port on the network host for the project. This allows users to have free access to the virtual machines in their project without exposing those machines to the public internet. + + +Cloudpipe Image +--------------- + +The cloudpipe image is basically just a linux instance with openvpn installed. It needs a simple script to grab user data from the metadata server, b64 decode it into a zip file, and run the autorun.sh script from inside the zip. The autorun script will configure and run openvpn to run using the data from nova. + +It is also useful to have a cron script that will periodically redownload the metadata and copy the new crl. This will keep revoked users from connecting and will disconnect any users that are connected with revoked certificates when their connection is renegotiated (every hour). + + +Cloudpipe Launch +---------------- + +When you use nova-manage to launch a cloudpipe for a user, it goes through the following process: + +#. creates a keypair called -vpn and saves it in the keys directory +#. creates a security group -vpn and opens up 1194 and icmp +#. creates a cert and private key for the vpn instance and saves it in the CA/projects// directory +#. zips up the info and puts it b64 encoded as user data +#. launches an m1.tiny instance with the above settings using the flag-specified vpn image + + +Vpn Access +---------- + +In vlan networking mode, the second ip in each private network is reserved for the cloudpipe instance. This gives a consistent ip to the instance so that nova-network can create forwarding rules for access from the outside world. The network for each project is given a specific high-numbered port on the public ip of the network host. This port is automatically forwarded to 1194 on the vpn instance. + +If specific high numbered ports do not work for your users, you can always allocate and associate a public ip to the instance, and then change the vpn_public_ip and vpn_public_port in the database. This will be turned into a nova-manage command or a flag soon. + + +Certificates and Revocation +--------------------------- + +If the use_project_ca flag is set (required to for cloudpipes to work securely), then each project has its own ca. This ca is used to sign the certificate for the vpn, and is also passed to the user for bundling images. When a certificate is revoked using nova-manage, a new Certificate Revocation List (crl) is generated. As long as cloudpipe has an updated crl, it will block revoked users from connecting to the vpn. + + +The :mod:`nova.cloudpipe.pipelib` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.cloudpipe.pipelib + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.api.cloudpipe` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.api.cloudpipe + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.crypto` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.crypto + :noindex: + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/devref/compute.rst b/doc/source/devref/compute.rst new file mode 100644 index 000000000000..db9ef6f3424b --- /dev/null +++ b/doc/source/devref/compute.rst @@ -0,0 +1,153 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +Virtualization +============== + + +Compute +------- + +Documentation for the compute manager and related files. For reading about +a specific virtualization backend, read Drivers_. + + +The :mod:`nova.compute.manager` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.manager + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.virt.connection` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.connection + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.compute.disk` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.disk + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.virt.images` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.images + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.compute.instance_types` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.instance_types + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.compute.power_state` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.power_state + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Drivers +------- + + +The :mod:`nova.virt.libvirt_conn` Driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.libvirt_conn + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.virt.xenapi` Driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.xenapi + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.virt.fake` Driver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.virt.fake + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Monitoring +---------- + +The :mod:`nova.compute.monitor` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.compute.monitor + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Tests +----- + +The :mod:`compute_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.compute_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`virt_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.virt_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/fakes.rst b/doc/source/devref/database.rst similarity index 54% rename from doc/source/fakes.rst rename to doc/source/devref/database.rst index a993fb4c89af..14559aa8c3ab 100644 --- a/doc/source/fakes.rst +++ b/doc/source/devref/database.rst @@ -15,29 +15,49 @@ License for the specific language governing permissions and limitations under the License. -Nova Fakes -========== +The Database Layer +================== -The :mod:`virt.fake` Module --------------------------- +The :mod:`nova.db.api` Module +----------------------------- -.. automodule:: nova.virt.fake +.. automodule:: nova.db.api + :noindex: :members: :undoc-members: :show-inheritance: - -The :mod:`fakeldap` Module --------------------------- -.. automodule:: nova.auth.fakeldap + +The Sqlalchemy Driver +--------------------- + +The :mod:`nova.db.sqlalchemy.api` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.db.sqlalchemy.api + :noindex: + +The :mod:`nova.db.sqlalchemy.models` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.db.sqlalchemy.models + :noindex: :members: :undoc-members: :show-inheritance: - -The :mod:`fakerabbit` Module ----------------------------- -.. automodule:: nova.fakerabbit +The :mod:`nova.db.sqlalchemy.session` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.db.sqlalchemy.session + :noindex: :members: :undoc-members: :show-inheritance: + + +Tests +----- + +Tests are lacking for the db api layer and for the sqlalchemy driver. +Failures in the drivers would be dectected in other test cases, though. diff --git a/doc/source/devref/development.environment.rst b/doc/source/devref/development.environment.rst new file mode 100644 index 000000000000..34104c96420c --- /dev/null +++ b/doc/source/devref/development.environment.rst @@ -0,0 +1,21 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Setting up a development environment +==================================== + +.. todo:: write this diff --git a/doc/source/endpoint.rst b/doc/source/devref/fakes.rst similarity index 53% rename from doc/source/endpoint.rst rename to doc/source/devref/fakes.rst index 399df416189e..0ba5d6ef2bec 100644 --- a/doc/source/endpoint.rst +++ b/doc/source/devref/fakes.rst @@ -15,77 +15,71 @@ License for the specific language governing permissions and limitations under the License. -Endpoint Documentation -====================== +Fake Drivers +============ -This page contains the Endpoint Package documentation. +.. todo:: document general info about fakes -The :mod:`admin` Module ------------------------ +When the real thing isn't available and you have some development to do these +fake implementations of various drivers let you get on with your day. -.. automodule:: nova.endpoint.admin - :members: - :undoc-members: - :show-inheritance: -The :mod:`api` Module ---------------------- - -.. automodule:: nova.endpoint.api - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud` Module ------------------------ - -.. automodule:: nova.endpoint.cloud - :members: - :undoc-members: - :show-inheritance: - -The :mod:`images` Module ------------------------- - -.. automodule:: nova.endpoint.images - :members: - :undoc-members: - :show-inheritance: - - -RELATED TESTS --------------- - -The :mod:`api_unittest` Module ------------------------------- - -.. automodule:: nova.tests.api_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`api_integration` Module ---------------------------------- - -.. automodule:: nova.tests.api_integration - :members: - :undoc-members: - :show-inheritance: - -The :mod:`cloud_unittest` Module +The :mod:`nova.virt.fake` Module -------------------------------- -.. automodule:: nova.tests.cloud_unittest - :members: - :undoc-members: - :show-inheritance: - -The :mod:`network_unittest` Module ----------------------------------- - -.. automodule:: nova.tests.network_unittest +.. automodule:: nova.virt.fake + :noindex: :members: :undoc-members: :show-inheritance: +The :mod:`nova.auth.fakeldap` Module +------------------------------------ + +.. automodule:: nova.auth.fakeldap + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.fakerabbit` Module +--------------------------------- + +.. automodule:: nova.fakerabbit + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :class:`nova.volume.driver.FakeAOEDriver` Class +--------------------------------------------------- + +.. autoclass:: nova.volume.driver.FakeAOEDriver + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :class:`nova.tests.service_unittest.FakeManager` Class +---------------------------------------------------------- + +.. autoclass:: nova.tests.service_unittest.FakeManager + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.tests.api.openstack.fakes` Module +------------------------------------------------ + +.. automodule:: nova.tests.api.openstack.fakes + :noindex: + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/storage.rst b/doc/source/devref/glance.rst similarity index 69% rename from doc/source/storage.rst rename to doc/source/devref/glance.rst index f77e5f0e523a..d18f7fec6787 100644 --- a/doc/source/storage.rst +++ b/doc/source/devref/glance.rst @@ -15,17 +15,14 @@ License for the specific language governing permissions and limitations under the License. -Storage in the Nova Cloud -========================= +Glance Integration - The Future of File Storage +=============================================== -There are three primary classes of storage in a nova cloud environment: +The :mod:`nova.image.service` Module +------------------------------------ -* Ephemeral Storage (local disk within an instance) -* Volume Storage (network-attached FS) -* Object Storage (redundant KVS with locality and MR) - -.. toctree:: - :maxdepth: 2 - - volume - objectstore +.. automodule:: nova.image.service + :noindex: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/devref/index.rst b/doc/source/devref/index.rst new file mode 100644 index 000000000000..6a93e3e188c8 --- /dev/null +++ b/doc/source/devref/index.rst @@ -0,0 +1,62 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Developer Guide +=============== + +In this section you will find information on Nova's lower level programming APIs. + + +Programming HowTos and Tutorials +-------------------------------- + +.. todo:: Add some programming howtos and tuts + +API Reference +------------- +.. toctree:: + :maxdepth: 3 + + ../api/autoindex + +Module Reference +---------------- +.. toctree:: + :maxdepth: 3 + + services + database + volume + compute + network + auth + api + scheduler + fakes + nova + cloudpipe + objectstore + glance + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/devref/modules.rst b/doc/source/devref/modules.rst new file mode 100644 index 000000000000..31792b219e36 --- /dev/null +++ b/doc/source/devref/modules.rst @@ -0,0 +1,19 @@ +Module Reference +================ + +.. toctree:: + :maxdepth: 1 + + services + database + volume + compute + network + auth + api + scheduler + fakes + nova + cloudpipe + objectstore + glance diff --git a/doc/source/network.rst b/doc/source/devref/network.rst similarity index 73% rename from doc/source/network.rst rename to doc/source/devref/network.rst index 357a0517f35a..d9d0914946f7 100644 --- a/doc/source/network.rst +++ b/doc/source/devref/network.rst @@ -1,6 +1,6 @@ .. Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. + Administrator of the National Aeronautics and Space Administration. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,8 +15,48 @@ License for the specific language governing permissions and limitations under the License. -nova Networking -================ +Networking +========== + +.. todo:: + + * document hardware specific commands (maybe in admin guide?) (todd) + * document a map between flags and managers/backends (todd) + + +The :mod:`nova.network.manager` Module +-------------------------------------- + +.. automodule:: nova.network.manager + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.network.linux_net` Driver +---------------------------------------- + +.. automodule:: nova.network.linux_net + :noindex: + :members: + :undoc-members: + :show-inheritance: + +Tests +----- + +The :mod:`network_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.network_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Legacy docs +----------- The nova networking components manage private networks, public IP addressing, VPN connectivity, and firewall rules. @@ -24,65 +64,65 @@ Components ---------- There are several key components: -* NetworkController (Manages address and vlan allocation) +* NetworkController (Manages address and vlan allocation) * RoutingNode (NATs public IPs to private IPs, and enforces firewall rules) * AddressingNode (runs DHCP services for private networks) * BridgingNode (a subclass of the basic nova ComputeNode) * TunnelingNode (provides VPN connectivity) - + Component Diagram ----------------- Overview:: - (PUBLIC INTERNET) + (PUBLIC INTERNET) | \ / \ / \ [RoutingNode] ... [RN] [TunnelingNode] ... [TN] | \ / | | | < AMQP > | | - [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] + [AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode] \ | \ / / \ / \ / \ / \ [BridgingNode] ... [BridgingNode] - - + + [NetworkController] ... [NetworkController] \ / < AMQP > | / \ - [CloudController]...[CloudController] + [CloudController]...[CloudController] -While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). +While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently). State Model ----------- Network State consists of the following facts: * VLAN assignment (to a project) -* Private Subnet assignment (to a security group) in a VLAN +* Private Subnet assignment (to a security group) in a VLAN * Private IP assignments (to running instances) * Public IP allocations (to a project) * Public IP associations (to a private IP / running instance) -While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. +While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash. The Public Traffic Path ----------------------- - + Public Traffic:: (PUBLIC INTERNET) | - <-- [RoutingNode] + <-- [RoutingNode] | [AddressingNode] --> | - ( VLAN ) + ( VLAN ) | <-- [BridgingNode] | - + -The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. +The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance. -Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). +Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details). diff --git a/doc/source/devref/nova.rst b/doc/source/devref/nova.rst new file mode 100644 index 000000000000..53ce6f34fc3c --- /dev/null +++ b/doc/source/devref/nova.rst @@ -0,0 +1,235 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Common and Misc Libraries +========================= + +Libraries common throughout Nova or just ones that haven't been categorized +very well yet. + + +The :mod:`nova.adminclient` Module +---------------------------------- + +.. automodule:: nova.adminclient + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.context` Module +------------------------------ + +.. automodule:: nova.context + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.exception` Module +-------------------------------- + +.. automodule:: nova.exception + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.flags` Module +---------------------------- + +.. automodule:: nova.flags + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.process` Module +------------------------------ + +.. automodule:: nova.process + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.rpc` Module +-------------------------- + +.. automodule:: nova.rpc + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.server` Module +----------------------------- + +.. automodule:: nova.server + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.test` Module +--------------------------- + +.. automodule:: nova.test + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.twistd` Module +----------------------------- + +.. automodule:: nova.twistd + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.utils` Module +---------------------------- + +.. automodule:: nova.utils + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.validate` Module +------------------------------- + +.. automodule:: nova.validate + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.wsgi` Module +--------------------------- + +.. automodule:: nova.wsgi + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Tests +----- + +The :mod:`declare_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.declare_flags + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`fake_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.fake_flags + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`flags_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.flags_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`process_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.process_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`real_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.real_flags + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`rpc_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.rpc_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`runtime_flags` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.runtime_flags + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`twistd_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.twistd_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`validator_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.validator_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/objectstore.rst b/doc/source/devref/objectstore.rst similarity index 71% rename from doc/source/objectstore.rst rename to doc/source/devref/objectstore.rst index 6b8d293f46f7..3ccfc8566fc6 100644 --- a/doc/source/objectstore.rst +++ b/doc/source/devref/objectstore.rst @@ -15,52 +15,57 @@ License for the specific language governing permissions and limitations under the License. -Objectstore Documentation -========================= +Objectstore - File Storage Service +================================== -This page contains the Objectstore Package documentation. - - -The :mod:`bucket` Module ------------------------- - -.. automodule:: nova.objectstore.bucket - :members: - :undoc-members: - :show-inheritance: - -The :mod:`handler` Module -------------------------- +The :mod:`nova.objectstore.handler` Module +------------------------------------------ .. automodule:: nova.objectstore.handler + :noindex: :members: :undoc-members: :show-inheritance: -The :mod:`image` Module ------------------------ -.. automodule:: nova.objectstore.image +The :mod:`nova.objectstore.bucket` Module +----------------------------------------- + +.. automodule:: nova.objectstore.bucket + :noindex: :members: :undoc-members: :show-inheritance: -The :mod:`stored` Module ------------------------- + +The :mod:`nova.objectstore.stored` Module +----------------------------------------- .. automodule:: nova.objectstore.stored + :noindex: :members: :undoc-members: :show-inheritance: -RELATED TESTS -------------- + +The :mod:`nova.objecstore.image` Module +---------------------------------------- + +.. automodule:: nova.objectstore.image + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Tests +----- The :mod:`objectstore_unittest` Module --------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: nova.tests.objectstore_unittest + :noindex: :members: :undoc-members: :show-inheritance: - diff --git a/doc/source/compute.rst b/doc/source/devref/scheduler.rst similarity index 59% rename from doc/source/compute.rst rename to doc/source/devref/scheduler.rst index 5b08dbd5b2af..ab74b6ba837e 100644 --- a/doc/source/compute.rst +++ b/doc/source/devref/scheduler.rst @@ -15,60 +15,57 @@ License for the specific language governing permissions and limitations under the License. -Compute Documentation -===================== +Scheduler +========= -This page contains the Compute Package documentation. +The :mod:`nova.scheduler.manager` Module +---------------------------------------- - -The :mod:`disk` Module ----------------------- - -.. automodule:: nova.compute.disk +.. automodule:: nova.scheduler.manager + :noindex: :members: :undoc-members: :show-inheritance: -The :mod:`exception` Module ---------------------------- -.. automodule:: nova.compute.exception +The :mod:`nova.scheduler.driver` Module +--------------------------------------- + +.. automodule:: nova.scheduler.driver + :noindex: :members: :undoc-members: :show-inheritance: -The :mod:`model` Module -------------------------- -.. automodule:: nova.compute.model - :members: - :undoc-members: - :show-inheritance: - -The :mod:`network` Module -------------------------- +The :mod:`nova.scheduler.chance` Driver +--------------------------------------- -.. automodule:: nova.compute.network +.. automodule:: nova.scheduler.chance + :noindex: :members: :undoc-members: :show-inheritance: -The :mod:`node` Module ----------------------- -.. automodule:: nova.compute.node - :members: - :undoc-members: - :show-inheritance: - -RELATED TESTS ---------------- +The :mod:`nova.scheduler.simple` Driver +--------------------------------------- -The :mod:`node_unittest` Module -------------------------------- - -.. automodule:: nova.tests.node_unittest +.. automodule:: nova.scheduler.simple + :noindex: :members: :undoc-members: :show-inheritance: + +Tests +----- + +The :mod:`scheduler_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.scheduler_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/devref/services.rst b/doc/source/devref/services.rst new file mode 100644 index 000000000000..f5bba5c126d5 --- /dev/null +++ b/doc/source/devref/services.rst @@ -0,0 +1,55 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _service_manager_driver: + +Services, Managers and Drivers +============================== + +The responsibilities of Services, Managers, and Drivers, can be a bit confusing to people that are new to nova. This document attempts to outline the division of responsibilities to make understanding the system a little bit easier. + +Currently, Managers and Drivers are specified by flags and loaded using utils.load_object(). This method allows for them to be implemented as singletons, classes, modules or objects. As long as the path specified by the flag leads to an object (or a callable that returns an object) that responds to getattr, it should work as a manager or driver. + + +The :mod:`nova.service` Module +------------------------------ + +.. automodule:: nova.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +The :mod:`nova.manager` Module +------------------------------ + +.. automodule:: nova.manager + :noindex: + :members: + :undoc-members: + :show-inheritance: + + +Implementation-Specific Drivers +------------------------------- + +A manager will generally load a driver for some of its tasks. The driver is responsible for specific implementation details. Anything running shell commands on a host, or dealing with other non-python code should probably be happening in a driver. + +Drivers should minimize touching the database, although it is currently acceptable for implementation specific data. This may be reconsidered at some point. + +It usually makes sense to define an Abstract Base Class for the specific driver (i.e. VolumeDriver), to define the methods that a different driver would need to implement. diff --git a/doc/source/volume.rst b/doc/source/devref/volume.rst similarity index 71% rename from doc/source/volume.rst rename to doc/source/devref/volume.rst index 619968458d43..54a2d4f8bccb 100644 --- a/doc/source/volume.rst +++ b/doc/source/devref/volume.rst @@ -15,9 +15,46 @@ License for the specific language governing permissions and limitations under the License. -Volume Documentation -==================== - +Storage Volumes, Disks +====================== + +.. todo:: rework after iSCSI merge (see 'Old Docs') (todd or vish) + + +The :mod:`nova.volume.manager` Module +------------------------------------- + +.. automodule:: nova.volume.manager + :noindex: + :members: + :undoc-members: + :show-inheritance: + +The :mod:`nova.volume.driver` Module +------------------------------------- + +.. automodule:: nova.volume.driver + :noindex: + :members: + :undoc-members: + :show-inheritance: + :exclude-members: FakeAOEDriver + +Tests +----- + +The :mod:`volume_unittest` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: nova.tests.volume_unittest + :noindex: + :members: + :undoc-members: + :show-inheritance: + +Old Docs +-------- + Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances. Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs. @@ -27,19 +64,3 @@ AoE exports are numbered according to a "shelf and blade" syntax. In order to av The underlying volumes are LVM logical volumes, created on demand within a single large volume group. -The :mod:`storage` Module -------------------------- - -.. automodule:: nova.volume.storage - :members: - :undoc-members: - :show-inheritance: - -The :mod:`storage_unittest` Module ----------------------------------- - -.. automodule:: nova.tests.storage_unittest - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst deleted file mode 100644 index 2df4a45ea51a..000000000000 --- a/doc/source/getting.started.rst +++ /dev/null @@ -1,122 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -Getting Started with Nova -========================= - -This code base is continually changing so dependencies also change. - -Dependencies ------------- - -Related servers we rely on - -* RabbitMQ: messaging queue, used for all communication between components - -Optional servers - -* OpenLDAP: By default, the auth server uses the RDBMS-backed datastore by setting FLAGS.auth_driver to 'nova.auth.dbdriver.DbDriver'. But OpenLDAP (or LDAP) could be configured. -* ReDIS: By default, this is not enabled as the auth driver. - -Python libraries we don't vendor - -* M2Crypto: python library interface for openssl -* curl -* XenAPI: Needed only for Xen Cloud Platform or XenServer support. Available from http://wiki.xensource.com/xenwiki/XCP_SDK or http://community.citrix.com/cdn/xs/sdks. - -Vendored python libaries (don't require any installation) - -* Twisted: just for the twisted.internet.defer package -* Tornado: scalable non blocking web server for api requests -* boto: python api for aws api -* IPy: library for managing ip addresses - -Recommended ------------------ - -* euca2ools: python implementation of aws ec2-tools and ami tools -* build tornado to use C module for evented section - - -Installation --------------- - - Due to many changes it's best to rely on the `OpenStack wiki `_ for installation instructions. - -Configuration ---------------- - -These instructions are incomplete, but we are actively updating the `OpenStack wiki `_ with more configuration information. - -On the cloud controller - -* Add yourself to the libvirtd group, log out, and log back in -* Fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances - -:: - - iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773 - iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE - - -* Configure NginX proxy (/etc/nginx/sites-enabled/default) - -:: - - server { - listen 3333 default; - server-name localhost; - client_max_body_size 10m; - - access_log /var/log/nginx/localhost.access.log; - - location ~ /_images/.+ { - root NOVA_PATH/images; - rewrite ^/_images/(.*)$ /$1 break; - } - - location / { - proxy_pass http://localhost:3334/; - } - } - -On the volume node - -* Create a filesystem (you can use an actual disk if you have one spare, default is /dev/sdb) - -:: - - # This creates a 1GB file to create volumes out of - dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10 - losetup --show -f MY_FILE_PATH - # replace loop0 below with whatever losetup returns - echo "--storage_dev=/dev/loop0" >> NOVA_PATH/bin/nova.conf - -Running ---------- - -Launch servers - -* rabbitmq -* redis (optional) - -Launch nova components - -* nova-api -* nova-compute -* nova-objectstore -* nova-volume diff --git a/doc/source/images/cloudpipe.png b/doc/source/images/cloudpipe.png new file mode 100644 index 000000000000..ffdd181f26a4 Binary files /dev/null and b/doc/source/images/cloudpipe.png differ diff --git a/doc/source/images/fabric.png b/doc/source/images/fabric.png new file mode 100644 index 000000000000..a5137e377f85 Binary files /dev/null and b/doc/source/images/fabric.png differ diff --git a/doc/source/index.rst b/doc/source/index.rst index 1109e9011830..9b2c8e1f8d3a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -18,37 +18,66 @@ Welcome to Nova's documentation! ================================ -Nova is a cloud computing fabric controller (the main part of an IaaS system). -It is written in Python and relies on the standard AMQP messaging protocol, uses the Twisted framework, -and optionally uses the Redis distributed key value store for authorization. +Nova is a cloud computing fabric controller, the main part of an IaaS system. +Individuals and organizations can use Nova to host and manage their own cloud +computing systems. Nova originated as a project out of NASA Ames Research Laboratory. -Nova is intended to be easy to extend and adapt. For example, authentication and authorization -requests by default use an RDBMS-backed datastore driver. However, there is already support -for using LDAP backing authentication (slapd) and if you wish to "fake" LDAP, there is a module -available that uses ReDIS to store authentication information in an LDAP-like backing datastore. -It has extensive test coverage, and uses the Sphinx toolkit (the same as Python itself) for code -and developer documentation. Additional documentation is available on the -'OpenStack wiki '_. -While Nova is currently in Beta use within several organizations, the codebase -is very much under active development - please test it and log bugs! +Nova is written with the following design guidelines in mind: -Contents: +* **Component based architecture**: Quickly add new behaviors +* **Highly available**: Scale to very serious workloads +* **Fault-Tollerant**: Isloated processes avoid cascading failures +* **Recoverable**: Failures should be easy to diagnose, debug, and rectify +* **Open Standards**: Be a reference implementation for a community-driven api +* **API Compatibility**: Nova strives to provide API-compatible with popular systems like Amazon EC2 + +This documentation is generated by the Sphinx toolkit and lives in the source +tree. Additional documentation on Nova and other components of OpenStack can +be found on the `OpenStack wiki`_. Also see the :doc:`community` page for +other ways to interact with the community. + +.. _`OpenStack wiki`: http://wiki.openstack.org + + +Key Concepts +============ +.. toctree:: + :maxdepth: 1 + + cloud101 + nova.concepts + swift.concepts + service.architecture + nova.object.model + swift.object.model + +Administrator's Documentation +============================= .. toctree:: - :maxdepth: 2 - - getting.started - architecture - network - storage - auth - compute - endpoint - nova - fakes - binaries - modules - packages + :maxdepth: 1 + + livecd + adminguide/index + adminguide/single.node.install + adminguide/multi.node.install + +.. todo:: add swiftadmin + +Developer Docs +============== + +.. toctree:: + :maxdepth: 1 + + quickstart + devref/index + community + +Outstanding Documentation Tasks +=============================== + +.. todolist:: Indices and tables ================== diff --git a/doc/source/installer.rst b/doc/source/installer.rst new file mode 100644 index 000000000000..b67e0e4f920c --- /dev/null +++ b/doc/source/installer.rst @@ -0,0 +1,12 @@ +Live CD +======= + +* 3 Images +* Once you start bundling images, must be able to point to source code +* Could make part of build + +* sudo nova-manage user admin newuser +* sudo nova-manage project create demo newuser +* sudo nova-manage project zipfile demo +* get images +* Web browser diff --git a/doc/source/livecd.rst b/doc/source/livecd.rst new file mode 100644 index 000000000000..82cf4658a027 --- /dev/null +++ b/doc/source/livecd.rst @@ -0,0 +1,2 @@ +Installing the Live CD +====================== diff --git a/doc/source/nova.concepts.rst b/doc/source/nova.concepts.rst new file mode 100644 index 000000000000..ddf0f1b829bd --- /dev/null +++ b/doc/source/nova.concepts.rst @@ -0,0 +1,203 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +Nova Concepts and Introduction +============================== + + +Introduction +------------ + +Nova is the software that controls your Infrastructure as as Service (IaaS) +cloud computing platform. It is similar in scope to Amazon EC2 and Rackspace +CloudServers. Nova does not include any virtualization software, rather it +defines drivers that interact with underlying virtualization mechanisms that +run on your host operating system, and exposes functionality over a web API. + +This document does not attempt to explain fundamental concepts of cloud +computing, IaaS, virtualization, or other related technologies. Instead, it +focuses on describing how Nova's implementation of those concepts is achieved. + +This page outlines concepts that you will need to understand as a user or +administrator of an OpenStack installation. Each section links to more more +detailed information in the :doc:`adminguide/index`, +but you'll probably want to read this section straight-through before tackling +the specifics presented in the administration guide. + + +Concept: Users and Projects +--------------------------- + +* access to images is limited by project +* access/secret are per user +* keypairs are per user +* quotas are per project + + +Concept: Virtualization +----------------------- + +* KVM +* UML +* XEN +* HyperV +* qemu + + +Concept: Instances +------------------ + +An 'instance' is a word for a virtual machine that runs inside the cloud. + +Concept: Storage +---------------- + +Volumes +~~~~~~~ + +A 'volume' is a detachable block storage device. You can think of it as a usb hard drive. It can only be attached to one instance at a time, so it does not work like a SAN. If you wish to expose the same volume to multiple instances, you will have to use an NFS or SAMBA share from an existing instance. + +Local Storage +~~~~~~~~~~~~~ + +Every instance larger than m1.tiny starts with some local storage (up to 160GB for m1.xlarge). This storage is currently the second partition on the root drive. + +Concept: Quotas +--------------- + +Nova supports per-project quotas. There are currently quotas for number of instances, total number of cores, number of volumes, total number of gigabytes, and number of floating ips. + + +Concept: RBAC +------------- + +Nova provides roles based access control (RBAC) for access to api commands. A user can have a number of different :ref:`roles `. Roles define which api_commands a user can perform. + +It is important to know that there are user-specific (sometimes called global) roles and project-specific roles. A user's actual permissions in a particular project are the INTERSECTION of his user-specific roles and is project-specific roles. + +For example: A user can access api commands allowed to the netadmin role (like allocate_address) only if he has the user-specific netadmin role AND the project-specific netadmin role. + +More information about RBAC can be found in the :ref:`auth`. + +Concept: API +------------ + +* EC2 +* OpenStack / Rackspace + + +Concept: Networking +------------------- + +Nova has a concept of Fixed Ips and Floating ips. Fixed ips are assigned to an instance on creation and stay the same until the instance is explicitly terminated. Floating ips are ip addresses that can be dynamically associated with an instance. This address can be disassociated and associated with another instance at any time. + +There are multiple strategies available for implementing fixed ips: + +Flat Mode +~~~~~~~~~ + +The simplest networking mode. Each instance receives a fixed ip from the pool. All instances are attached to the same bridge (br100) by default. The bridge must be configured manually. The networking configuration is injected into the instance before it is booted. Note that this currently only works on linux-style systems that keep networking configuration in /etc/network/interfaces. + +Flat DHCP Mode +~~~~~~~~~~~~~~ + +This is similar to the flat mode, in that all instances are attached to the same bridge. In this mode nova does a bit more configuration, it will attempt to bridge into an ethernet device (eth0 by default). It will also run dnsmasq as a dhcpserver listening on this bridge. Instances receive their fixed ips by doing a dhcpdiscover. + +VLAN DHCP Mode +~~~~~~~~~~~~~~ + +This is the default networking mode and supports the most features. For multiple machine installation, it requires a switch that supports host-managed vlan tagging. In this mode, nova will create a vlan and bridge for each project. The project gets a range of private ips that are only accessible from inside the vlan. In order for a user to access the instances in their project, a special vpn instance (code named :ref:`cloudpipe `) needs to be created. Nova generates a certificate and key for the user to access the vpn and starts the vpn automatically. More information on cloudpipe can be found :ref:`here `. + +The following diagram illustrates how the communication that occurs between the vlan (the dashed box) and the public internet (represented by the two clouds) + +.. image:: /images/cloudpipe.png + :width: 100% + +.. + +Concept: Binaries +----------------- + +Nova is implemented by a number of related binaries. These binaries can run on the same machine or many machines. A detailed description of each binary is given in the :ref:`binaries section ` of the developer guide. + +.. _manage_usage: + +Concept: nova-manage +-------------------- + +The nova-manage command is used to perform many essential functions for +administration and ongoing maintenance of nova, such as user creation, +vpn management, and much more. + +See doc:`nova.manage` in the Administration Guide for more details. + + +Concept: Flags +-------------- + +python-gflags + + +Concept: Plugins +---------------- + +* Managers/Drivers: utils.import_object from string flag +* virt/connections: conditional loading from string flag +* db: LazyPluggable via string flag +* auth_manager: utils.import_class based on string flag +* Volumes: moving to pluggable driver instead of manager +* Network: pluggable managers +* Compute: same driver used, but pluggable at connection + + +Concept: IPC/RPC +---------------- + +Nova utilizes the RabbitMQ implementation of the AMQP messaging standard for performing communication between the various nova services. This message queuing service is used for both local and remote communication because Nova is designed so that there is no requirement that any of the services exist on the same physical machine. RabbitMQ in particular is very robust and provides the efficiency and reliability that Nova needs. More information about RabbitMQ can be found at http://www.rabbitmq.com/. + +Concept: Fakes +-------------- + +* auth +* ldap + + +Concept: Scheduler +------------------ + +* simple +* random + + +Concept: Security Groups +------------------------ + +Security groups + + +Concept: Certificate Authority +------------------------------ + +Nova does a small amount of certificate management. These certificates are used for :ref:`project vpns ` and decrypting bundled images. + + +Concept: Images +--------------- + +* launching +* bundling diff --git a/doc/source/nova.rst b/doc/source/nova.rst deleted file mode 100644 index 4b9c44a5f027..000000000000 --- a/doc/source/nova.rst +++ /dev/null @@ -1,91 +0,0 @@ -.. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -NOVA Libraries -=============== - -The :mod:`crypto` Module ------------------------- - -.. automodule:: nova.crypto - :members: - :undoc-members: - :show-inheritance: - -The :mod:`adminclient` Module ------------------------------ - -.. automodule:: nova.adminclient - :members: - :undoc-members: - :show-inheritance: - -The :mod:`datastore` Module ---------------------------- - -.. automodule:: nova.datastore - :members: - :undoc-members: - :show-inheritance: - -The :mod:`exception` Module ---------------------------- - -.. automodule:: nova.exception - :members: - :undoc-members: - :show-inheritance: - -The :mod:`flags` Module ---------------------------- - -.. automodule:: nova.flags - :members: - :undoc-members: - :show-inheritance: - -The :mod:`rpc` Module ---------------------------- - -.. automodule:: nova.rpc - :members: - :undoc-members: - :show-inheritance: - -The :mod:`server` Module ---------------------------- - -.. automodule:: nova.server - :members: - :undoc-members: - :show-inheritance: - -The :mod:`test` Module ---------------------------- - -.. automodule:: nova.test - :members: - :undoc-members: - :show-inheritance: - -The :mod:`utils` Module ---------------------------- - -.. automodule:: nova.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/object.model.rst b/doc/source/object.model.rst new file mode 100644 index 000000000000..c8d4df7367fe --- /dev/null +++ b/doc/source/object.model.rst @@ -0,0 +1,53 @@ +Object Model +============ + +.. todo:: Add brief description for core models + +.. graphviz:: + + digraph foo { + graph [rankdir="LR"]; node [fontsize=9 shape=box]; + Instances -> "Public IPs" [arrowhead=crow]; + Instances -> "Security Groups" [arrowhead=crow]; + Users -> Projects [arrowhead=crow arrowtail=crow dir=both]; + Users -> Keys [arrowhead=crow]; + Instances -> Volumes [arrowhead=crow]; + Projects -> "Public IPs" [arrowhead=crow]; + Projects -> Instances [arrowhead=crow]; + Projects -> Volumes [arrowhead=crow]; + Projects -> Images [arrowhead=crow]; + Images -> Instances [arrowhead=crow]; + Projects -> "Security Groups" [arrowhead=crow]; + "Security Groups" -> Rules [arrowhead=crow]; + } + + +Users +----- + +Projects +-------- + + +Images +------ + + +Instances +--------- + + +Volumes +------- + + +Security Groups +--------------- + + +VLANs +----- + + +IP Addresses +------------ diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst new file mode 100644 index 000000000000..ae2b64d8a51d --- /dev/null +++ b/doc/source/quickstart.rst @@ -0,0 +1,178 @@ +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Nova Quickstart +=============== + +.. todo:: + P1 (this is one example of how to use priority syntax) + * Document the assumptions about pluggable interfaces (sqlite3 instead of + mysql, etc) (todd) + * Document env vars that can change things (USE_MYSQL, HOST_IP) (todd) + +Recommended System Configuration +-------------------------------- + +Although Nova can be run on a variety of system architectures, for most users the following will be simplest: + +* Ubuntu Lucid +* 10GB Hard Disk Space +* 512MB RAM + +For development, Nova can run from within a VM. + + +Getting the Code +---------------- + +Nova is hosted on launchpad. You can get the code with the following command + +:: + + bzr clone lp:nova + +The `contrib/nova.sh` file in the source distribution is a script that +will quickly set up nova to run on a single machine. It is tested against +Ubuntu only, but other distributions are forthcoming. + +Environment Variables +--------------------- + +By tweaking the environment that nova.sh run in, you can build slightly +different configurations (though for more complex setups you should see +:doc:`/adminguide/getting.started` and :doc:`/adminguide/multi.node.install`). + +* HOST_IP + * Default: address of first interface from the ifconfig command + * Values: 127.0.0.1, or any other valid address + +TEST +~~~~ + +**Default**: 0 +**Values**: 1, run tests after checkout and initial setup + +USE_MYSQL +~~~~~~~~~ + +**Default**: 0, use sqlite3 +**Values**: 1, use mysql instead of sqlite3 + +MYSQL_PASS +~~~~~~~~~~ + +Only useful if $USE_MYSQL=1. + +**Default**: nova +**Values**: value of root password for mysql + +USE_LDAP +~~~~~~~~ + +**Default**: 0, use :mod:`nova.auth.dbdriver` +**Values**: 1, use :mod:`nova.auth.ldapdriver` + +LIBVIRT_TYPE +~~~~~~~~~~~~ + +**Default**: qemu +**Values**: uml, kvm + +Usage +----- + +Unless you want to spend a lot of time fiddling with permissions and sudoers, +you should probably run nova as root. + +:: + + sudo -i + +If you are concerned about security, nova runs just fine inside a virtual +machine. + +Use the script to install and run the current trunk. You can also specify a +specific branch by putting `lp:~someone/nova/some-branch` after the branch +command + +:: + + ./nova.sh branch + ./nova.sh install + ./nova.sh run + +The run command will drop you into a screen session with all of the workers +running in different windows You can use eucatools to run commands against the +cloud. + +:: + + euca-add-keypair test > test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + euca-describe-instances + +To see output from the various workers, switch screen windows + +:: + + " + +will give you a list of running windows. + +When the instance is running, you should be able to ssh to it. + +:: + + chmod 600 test.pem + ssh -i test.pem root@10.0.0.3 + +When you exit screen + +:: + + + +nova will terminate. It may take a while for nova to finish cleaning up. If +you exit the process before it is done because there were some problems in your +build, you may have to clean up the nova processes manually. If you had any +instances running, you can attempt to kill them through the api: + +:: + + ./nova.sh terminate + +Then you can destroy the screen: + +:: + + ./nova.sh clean + +If things get particularly messed up, you might need to do some more intense +cleanup. Be careful, the following command will manually destroy all runnning +virsh instances and attempt to delete all vlans and bridges. + +:: + + ./nova.sh scrub + +You can edit files in the install directory or do a bzr pull to pick up new versions. You only need to do + +:: + + ./nova.sh run + +to run nova after the first install. The database should be cleaned up on each run. \ No newline at end of file diff --git a/doc/source/service.architecture.rst b/doc/source/service.architecture.rst new file mode 100644 index 000000000000..28a32bec6b56 --- /dev/null +++ b/doc/source/service.architecture.rst @@ -0,0 +1,60 @@ +Service Architecture +==================== + +Nova’s Cloud Fabric is composed of the following major components: + +* API Server +* Message Queue +* Compute Worker +* Network Controller +* Volume Worker +* Scheduler +* Image Store + + +.. image:: /images/fabric.png + :width: 790 + +API Server +-------------------------------------------------- +At the heart of the cloud framework is an API Server. This API Server makes command and control of the hypervisor, storage, and networking programmatically available to users in realization of the definition of cloud computing. + +The API endpoints are basic http web services which handle authentication, authorization, and basic command and control functions using various API interfaces under the Amazon, Rackspace, and related models. This enables API compatibility with multiple existing tool sets created for interaction with offerings from other vendors. This broad compatibility prevents vendor lock-in. + +Message Queue +-------------------------------------------------- +A messaging queue brokers the interaction between compute nodes (processing), volumes (block storage), the networking controllers (software which controls network infrastructure), API endpoints, the scheduler (determines which physical hardware to allocate to a virtual resource), and similar components. Communication to and from the cloud controller is by HTTP requests through multiple API endpoints. + +A typical message passing event begins with the API server receiving a request from a user. The API server authenticates the user and ensures that the user is permitted to issue the subject command. Availability of objects implicated in the request is evaluated and, if available, the request is routed to the queuing engine for the relevant workers. Workers continually listen to the queue based on their role, and occasionally their type hostname. When such listening produces a work request, the worker takes assignment of the task and begins its execution. Upon completion, a response is dispatched to the queue which is received by the API server and relayed to the originating user. Database entries are queried, added, or removed as necessary throughout the process. + +Compute Worker +-------------------------------------------------- +Compute workers manage computing instances on host machines. Through the API, commands are dispatched to compute workers to: + +* Run instances +* Terminate instances +* Reboot instances +* Attach volumes +* Detach volumes +* Get console output + +Network Controller +-------------------------------------------------- +The Network Controller manages the networking resources on host machines. The API server dispatches commands through the message queue, which are subsequently processed by Network Controllers. Specific operations include: + +* Allocate Fixed IP Addresses +* Configuring VLANs for projects +* Configuring networks for compute nodes + +Volume Workers +-------------------------------------------------- +Volume Workers interact with iSCSI storage to manage LVM-based instance volumes. Specific functions include: + +* Create Volumes +* Delete Volumes +* Establish Compute volumes + +Volumes may easily be transferred between instances, but may be attached to only a single instance at a time. + + +.. todo:: P2: image store description diff --git a/nova/adminclient.py b/nova/adminclient.py index b7a3d2c32431..af55197fc198 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -33,14 +33,15 @@ DEFAULT_SECRET_KEY = 'admin' class UserInfo(object): """ - Information about a Nova user, as parsed through SAX - fields include: - username - accesskey - secretkey + Information about a Nova user, as parsed through SAX. + + **Fields Include** + + * username + * accesskey + * secretkey + * file (optional) containing zip of X509 cert & rc file - and an optional field containing a zip with X509 cert & rc - file """ def __init__(self, connection=None, username=None, endpoint=None): @@ -68,9 +69,13 @@ class UserInfo(object): class UserRole(object): """ Information about a Nova user's role, as parsed through SAX. - Fields include: - role + + **Fields include** + + * role + """ + def __init__(self, connection=None): self.connection = connection self.role = None @@ -90,12 +95,15 @@ class UserRole(object): class ProjectInfo(object): """ - Information about a Nova project, as parsed through SAX - Fields include: - projectname - description - projectManagerId - memberIds + Information about a Nova project, as parsed through SAX. + + **Fields include** + + * projectname + * description + * projectManagerId + * memberIds + """ def __init__(self, connection=None): @@ -127,8 +135,11 @@ class ProjectInfo(object): class ProjectMember(object): """ Information about a Nova project member, as parsed through SAX. - Fields include: - memberId + + **Fields include** + + * memberId + """ def __init__(self, connection=None): @@ -150,14 +161,18 @@ class ProjectMember(object): class HostInfo(object): """ - Information about a Nova Host, as parsed through SAX: - Disk stats - Running Instances - Memory stats - CPU stats - Network address info - Firewall info - Bridge and devices + Information about a Nova Host, as parsed through SAX. + + **Fields Include** + + * Disk stats + * Running Instances + * Memory stats + * CPU stats + * Network address info + * Firewall info + * Bridge and devices + """ def __init__(self, connection=None): @@ -257,9 +272,12 @@ class NovaAdminClient(object): [('item', UserRole)]) def get_user_roles(self, user, project=None): - """Returns a list of roles for the given user. Omitting project will - return any global roles that the user has. Specifying project will - return only project specific roles.""" + """Returns a list of roles for the given user. + + Omitting project will return any global roles that the user has. + Specifying project will return only project specific roles. + + """ params = {'User': user} if project: params['Project'] = project diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 707c1623ea03..7e75445a8831 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -15,15 +15,22 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ Root WSGI middleware for all API controllers. + +**Related Flags** + +:osapi_subdomain: subdomain running the OpenStack API (default: api) +:ec2api_subdomain: subdomain running the EC2 API (default: ec2) +:FAKE_subdomain: set to 'api' or 'ec2', requests default to that endpoint + """ import routes import webob.dec from nova import flags +from nova import utils from nova import wsgi from nova.api import cloudpipe from nova.api import ec2 diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index c53ce6f5e8a2..b7664ec71da4 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -15,8 +15,10 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +""" +Starting point for routing EC2 requests. -"""Starting point for routing EC2 requests""" +""" import logging import routes diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index cf3a84a5d9d0..46e0135b4c65 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -15,12 +15,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" -Fake LDAP server for test harnesses. +"""Fake LDAP server for test harness, backs to ReDIS. This class does very little error checking, and knows nothing about ldap -class definitions. It implements the minimum emulation of the python ldap +class definitions. It implements the minimum emulation of the python ldap library to work with nova. + """ import json @@ -77,9 +77,8 @@ def initialize(_uri): def _match_query(query, attrs): """Match an ldap query to an attribute dictionary. - &, |, and ! are supported in the query. No syntax checking is performed, - so malformed querys will not work correctly. - + The characters &, |, and ! are supported in the query. No syntax checking + is performed, so malformed querys will not work correctly. """ # cut off the parentheses inner = query[1:-1] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 001a96875268..7b2b6816167c 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -84,12 +84,11 @@ class AuthBase(object): @classmethod def safe_id(cls, obj): - """Safe get object id + """Safely get object id. This method will return the id of the object if the object is of this class, otherwise it will return the original object. This allows methods to accept objects or ids as paramaters. - """ if isinstance(obj, cls): return obj.id diff --git a/nova/compute/disk.py b/nova/compute/disk.py index e362b4507e07..0b8568d33c13 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -15,10 +15,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ Utility methods to resize, repartition, and modify disk images. + Includes injection of SSH PGP keys into authorized_keys file. + """ import logging @@ -41,20 +42,23 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, @defer.inlineCallbacks def partition(infile, outfile, local_bytes=0, resize=True, local_type='ext2', execute=None): - """Takes a single partition represented by infile and writes a bootable - drive image into outfile. + """ + Turns a partition (infile) into a bootable drive image (outfile). The first 63 sectors (0-62) of the resulting image is a master boot record. Infile becomes the first primary partition. If local bytes is specified, a second primary partition is created and formatted as ext2. - In the diagram below, dashes represent drive sectors. - +-----+------. . .-------+------. . .------+ - | 0 a| b c|d e| - +-----+------. . .-------+------. . .------+ - | mbr | primary partiton | local partition | - +-----+------. . .-------+------. . .------+ + :: + + In the diagram below, dashes represent drive sectors. + +-----+------. . .-------+------. . .------+ + | 0 a| b c|d e| + +-----+------. . .-------+------. . .------+ + | mbr | primary partiton | local partition | + +-----+------. . .-------+------. . .------+ + """ sector_size = 512 file_size = os.path.getsize(infile) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 850cded8aa5b..890d79fba92e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -17,7 +17,21 @@ # under the License. """ -Handles all code relating to instances (guest vms) +Handles all processes relating to instances (guest vms). + +The :py:class:`ComputeManager` class is a :py:class:`nova.manager.Manager` that +handles RPC calls relating to creating instances. It is responsible for +building a disk image, launching it via the underlying virtualization driver, +responding to calls to check it state, attaching persistent as well as +termination. + +**Related Flags** + +:instances_path: Where instances are kept on disk +:compute_driver: Name of class that is used to handle virtualization, loaded + by :func:`nova.utils.import_object` +:volume_manager: Name of class that handles persistent storage, loaded by + :func:`nova.utils.import_object` """ import datetime @@ -40,12 +54,12 @@ flags.DEFINE_string('compute_driver', 'nova.virt.connection.get_connection', class ComputeManager(manager.Manager): - """ - Manages the running instances. - """ + """Manages the running instances from creation to destruction.""" + def __init__(self, compute_driver=None, *args, **kwargs): """Load configuration options and connect to the hypervisor.""" # TODO(vish): sync driver creation logic with the rest of the system + # and redocument the module docstring if not compute_driver: compute_driver = FLAGS.compute_driver self.driver = utils.import_object(compute_driver) @@ -54,7 +68,7 @@ class ComputeManager(manager.Manager): super(ComputeManager, self).__init__(*args, **kwargs) def _update_state(self, context, instance_id): - """Update the state of an instance from the driver info""" + """Update the state of an instance from the driver info.""" # FIXME(ja): include other fields from state? instance_ref = self.db.instance_get(context, instance_id) try: @@ -67,6 +81,7 @@ class ComputeManager(manager.Manager): @defer.inlineCallbacks @exception.wrap_exception def refresh_security_group(self, context, security_group_id, **_kwargs): + """This call passes stright through to the virtualization driver.""" yield self.driver.refresh_security_group(security_group_id) def create_instance(self, context, security_groups=None, **kwargs): @@ -76,9 +91,9 @@ class ComputeManager(manager.Manager): :param context: The security context :param security_groups: list of security group ids to attach to the instance - :param **kwargs: All additional keyword args are treated - as data fields of the instance to be - created + :param kwargs: All additional keyword args are treated + as data fields of the instance to be + created :retval Returns a mapping of the instance information that has just been created @@ -97,13 +112,13 @@ class ComputeManager(manager.Manager): return instance_ref def update_instance(self, context, instance_id, **kwargs): - """Updates the instance in the datastore + """Updates the instance in the datastore. :param context: The security context :param instance_id: ID of the instance to update - :param **kwargs: All additional keyword args are treated - as data fields of the instance to be - updated + :param kwargs: All additional keyword args are treated + as data fields of the instance to be + updated :retval None diff --git a/nova/crypto.py b/nova/crypto.py index 16b4f5e1f26a..d73559587f50 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -15,10 +15,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ -Wrappers around standard crypto, including root and intermediate CAs, -SSH key_pairs and x509 certificates. +Wrappers around standard crypto data elements. + +Includes root and intermediate CAs, SSH key_pairs and x509 certificates. + """ import base64 @@ -227,12 +228,12 @@ def mkcacert(subject='nova', years=1): def compute_md5(fp): """ - @type fp: file - @param fp: File pointer to the file to MD5 hash. The file pointer will be + :type fp: file + :param fp: File pointer to the file to MD5 hash. The file pointer will be reset to the beginning of the file before the method returns. - @rtype: tuple - @return: the hex digest version of the MD5 hash + :rtype: tuple + :return: the hex digest version of the MD5 hash """ m = hashlib.md5() fp.seek(0) diff --git a/nova/db/api.py b/nova/db/api.py index 80563c4529b9..8f9dc244301c 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -16,7 +16,17 @@ # License for the specific language governing permissions and limitations # under the License. """ -Defines interface for DB access +Defines interface for DB access. + +The underlying driver is loaded as a :class:`LazyPluggable`. + +**Related Flags** + +:db_backend: string to lookup in the list of LazyPluggable backends. + `sqlalchemy` is the only supported backend right now. + +:sql_connection: string specifying the sqlalchemy connection to use, like: + `sqlite:///var/lib/nova/nova.sqlite`. """ from nova import exception @@ -34,17 +44,17 @@ IMPL = utils.LazyPluggable(FLAGS['db_backend'], class NoMoreAddresses(exception.Error): - """No more available addresses""" + """No more available addresses.""" pass class NoMoreBlades(exception.Error): - """No more available blades""" + """No more available blades.""" pass class NoMoreNetworks(exception.Error): - """No more available networks""" + """No more available networks.""" pass @@ -67,30 +77,33 @@ def service_get(context, service_id): def service_get_all_by_topic(context, topic): - """Get all compute services for a given topic """ + """Get all compute services for a given topic.""" return IMPL.service_get_all_by_topic(context, topic) def service_get_all_compute_sorted(context): - """Get all compute services sorted by instance count + """Get all compute services sorted by instance count. + + Returns a list of (Service, instance_count) tuples. - Returns a list of (Service, instance_count) tuples """ return IMPL.service_get_all_compute_sorted(context) def service_get_all_network_sorted(context): - """Get all network services sorted by network count + """Get all network services sorted by network count. + + Returns a list of (Service, network_count) tuples. - Returns a list of (Service, network_count) tuples """ return IMPL.service_get_all_network_sorted(context) def service_get_all_volume_sorted(context): - """Get all volume services sorted by volume count + """Get all volume services sorted by volume count. + + Returns a list of (Service, volume_count) tuples. - Returns a list of (Service, volume_count) tuples """ return IMPL.service_get_all_volume_sorted(context) @@ -121,6 +134,7 @@ def floating_ip_allocate_address(context, host, project_id): """Allocate free floating ip and return the address. Raises if one is not available. + """ return IMPL.floating_ip_allocate_address(context, host, project_id) @@ -149,6 +163,7 @@ def floating_ip_disassociate(context, address): """Disassociate an floating ip from a fixed ip by address. Returns the address of the existing fixed ip. + """ return IMPL.floating_ip_disassociate(context, address) @@ -187,6 +202,7 @@ def fixed_ip_associate(context, address, instance_id): """Associate fixed ip to instance. Raises if fixed ip is not available. + """ return IMPL.fixed_ip_associate(context, address, instance_id) @@ -195,6 +211,7 @@ def fixed_ip_associate_pool(context, network_id, instance_id): """Find free ip in network and associate it to instance. Raises if one is not available. + """ return IMPL.fixed_ip_associate_pool(context, network_id, instance_id) @@ -210,7 +227,7 @@ def fixed_ip_disassociate(context, address): def fixed_ip_disassociate_all_by_timeout(context, host, time): - """Disassociate old fixed ips from host""" + """Disassociate old fixed ips from host.""" return IMPL.fixed_ip_disassociate_all_by_timeout(context, host, time) @@ -288,7 +305,7 @@ def instance_get_floating_address(context, instance_id): def instance_get_by_internal_id(context, internal_id): - """Get an instance by ec2 id.""" + """Get an instance by internal id.""" return IMPL.instance_get_by_internal_id(context, internal_id) @@ -312,7 +329,7 @@ def instance_update(context, instance_id, values): def instance_add_security_group(context, instance_id, security_group_id): - """Associate the given security group with the given instance""" + """Associate the given security group with the given instance.""" return IMPL.instance_add_security_group(context, instance_id, security_group_id) @@ -374,10 +391,12 @@ def network_count_reserved_ips(context, network_id): def network_create_safe(context, values): - """Create a network from the values dict + """Create a network from the values dict. The network is only returned if the create succeeds. If the create violates - constraints because the network already exists, no exception is raised.""" + constraints because the network already exists, no exception is raised. + + """ return IMPL.network_create_safe(context, values) @@ -418,22 +437,22 @@ def network_get_by_instance(context, instance_id): def network_get_index(context, network_id): - """Get non-conflicting index for network""" + """Get non-conflicting index for network.""" return IMPL.network_get_index(context, network_id) def network_get_vpn_ip(context, network_id): - """Get non-conflicting index for network""" + """Get non-conflicting index for network.""" return IMPL.network_get_vpn_ip(context, network_id) def network_set_cidr(context, network_id, cidr): - """Set the Classless Inner Domain Routing for the network""" + """Set the Classless Inner Domain Routing for the network.""" return IMPL.network_set_cidr(context, network_id, cidr) def network_set_host(context, network_id, host_id): - """Safely set the host for network""" + """Safely set the host for network.""" return IMPL.network_set_host(context, network_id, host_id) @@ -479,7 +498,9 @@ def export_device_create_safe(context, values): The device is not returned. If the create violates the unique constraints because the shelf_id and blade_id already exist, - no exception is raised.""" + no exception is raised. + + """ return IMPL.export_device_create_safe(context, values) @@ -504,17 +525,17 @@ def iscsi_target_create_safe(context, values): def auth_destroy_token(context, token): - """Destroy an auth token""" + """Destroy an auth token.""" return IMPL.auth_destroy_token(context, token) def auth_get_token(context, token_hash): - """Retrieves a token given the hash representing it""" + """Retrieves a token given the hash representing it.""" return IMPL.auth_get_token(context, token_hash) def auth_create_token(context, token): - """Creates a new token""" + """Creates a new token.""" return IMPL.auth_create_token(context, token) @@ -632,47 +653,47 @@ def volume_update(context, volume_id, values): def security_group_get_all(context): - """Get all security groups""" + """Get all security groups.""" return IMPL.security_group_get_all(context) def security_group_get(context, security_group_id): - """Get security group by its internal id""" + """Get security group by its internal id.""" return IMPL.security_group_get(context, security_group_id) def security_group_get_by_name(context, project_id, group_name): - """Returns a security group with the specified name from a project""" + """Returns a security group with the specified name from a project.""" return IMPL.security_group_get_by_name(context, project_id, group_name) def security_group_get_by_project(context, project_id): - """Get all security groups belonging to a project""" + """Get all security groups belonging to a project.""" return IMPL.security_group_get_by_project(context, project_id) def security_group_get_by_instance(context, instance_id): - """Get security groups to which the instance is assigned""" + """Get security groups to which the instance is assigned.""" return IMPL.security_group_get_by_instance(context, instance_id) def security_group_exists(context, project_id, group_name): - """Indicates if a group name exists in a project""" + """Indicates if a group name exists in a project.""" return IMPL.security_group_exists(context, project_id, group_name) def security_group_create(context, values): - """Create a new security group""" + """Create a new security group.""" return IMPL.security_group_create(context, values) def security_group_destroy(context, security_group_id): - """Deletes a security group""" + """Deletes a security group.""" return IMPL.security_group_destroy(context, security_group_id) def security_group_destroy_all(context): - """Deletes a security group""" + """Deletes a security group.""" return IMPL.security_group_destroy_all(context) @@ -680,18 +701,18 @@ def security_group_destroy_all(context): def security_group_rule_create(context, values): - """Create a new security group""" + """Create a new security group.""" return IMPL.security_group_rule_create(context, values) def security_group_rule_get_by_security_group(context, security_group_id): - """Get all rules for a a given security group""" + """Get all rules for a a given security group.""" return IMPL.security_group_rule_get_by_security_group(context, security_group_id) def security_group_rule_destroy(context, security_group_rule_id): - """Deletes a security group rule""" + """Deletes a security group rule.""" return IMPL.security_group_rule_destroy(context, security_group_rule_id) @@ -699,107 +720,107 @@ def security_group_rule_destroy(context, security_group_rule_id): def user_get(context, id): - """Get user by id""" + """Get user by id.""" return IMPL.user_get(context, id) def user_get_by_uid(context, uid): - """Get user by uid""" + """Get user by uid.""" return IMPL.user_get_by_uid(context, uid) def user_get_by_access_key(context, access_key): - """Get user by access key""" + """Get user by access key.""" return IMPL.user_get_by_access_key(context, access_key) def user_create(context, values): - """Create a new user""" + """Create a new user.""" return IMPL.user_create(context, values) def user_delete(context, id): - """Delete a user""" + """Delete a user.""" return IMPL.user_delete(context, id) def user_get_all(context): - """Create a new user""" + """Create a new user.""" return IMPL.user_get_all(context) def user_add_role(context, user_id, role): - """Add another global role for user""" + """Add another global role for user.""" return IMPL.user_add_role(context, user_id, role) def user_remove_role(context, user_id, role): - """Remove global role from user""" + """Remove global role from user.""" return IMPL.user_remove_role(context, user_id, role) def user_get_roles(context, user_id): - """Get global roles for user""" + """Get global roles for user.""" return IMPL.user_get_roles(context, user_id) def user_add_project_role(context, user_id, project_id, role): - """Add project role for user""" + """Add project role for user.""" return IMPL.user_add_project_role(context, user_id, project_id, role) def user_remove_project_role(context, user_id, project_id, role): - """Remove project role from user""" + """Remove project role from user.""" return IMPL.user_remove_project_role(context, user_id, project_id, role) def user_get_roles_for_project(context, user_id, project_id): - """Return list of roles a user holds on project""" + """Return list of roles a user holds on project.""" return IMPL.user_get_roles_for_project(context, user_id, project_id) def user_update(context, user_id, values): - """Update user""" + """Update user.""" return IMPL.user_update(context, user_id, values) def project_get(context, id): - """Get project by id""" + """Get project by id.""" return IMPL.project_get(context, id) def project_create(context, values): - """Create a new project""" + """Create a new project.""" return IMPL.project_create(context, values) def project_add_member(context, project_id, user_id): - """Add user to project""" + """Add user to project.""" return IMPL.project_add_member(context, project_id, user_id) def project_get_all(context): - """Get all projects""" + """Get all projects.""" return IMPL.project_get_all(context) def project_get_by_user(context, user_id): - """Get all projects of which the given user is a member""" + """Get all projects of which the given user is a member.""" return IMPL.project_get_by_user(context, user_id) def project_remove_member(context, project_id, user_id): - """Remove the given user from the given project""" + """Remove the given user from the given project.""" return IMPL.project_remove_member(context, project_id, user_id) def project_update(context, project_id, values): - """Update Remove the given user from the given project""" + """Update Remove the given user from the given project.""" return IMPL.project_update(context, project_id, values) def project_delete(context, project_id): - """Delete project""" + """Delete project.""" return IMPL.project_delete(context, project_id) @@ -808,6 +829,7 @@ def project_delete(context, project_id): def host_get_networks(context, host): """Return all networks for which the given host is the designated - network host + network host. + """ return IMPL.host_get_networks(context, host) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index db4d9f68f3d6..b8f999af4753 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. """ -Implementation of SQLAlchemy backend +Implementation of SQLAlchemy backend. """ import random diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 1111b5cbd7d4..01b5cf350710 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -15,9 +15,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ -SQLAlchemy models for nova data +SQLAlchemy models for nova data. """ import datetime @@ -35,13 +34,13 @@ from nova import auth from nova import exception from nova import flags -FLAGS = flags.FLAGS +FLAGS = flags.FLAGS BASE = declarative_base() class NovaBase(object): - """Base class for Nova Models""" + """Base class for Nova Models.""" __table_args__ = {'mysql_engine': 'InnoDB'} __table_initialized__ = False created_at = Column(DateTime, default=datetime.datetime.utcnow) @@ -50,7 +49,7 @@ class NovaBase(object): deleted = Column(Boolean, default=False) def save(self, session=None): - """Save this object""" + """Save this object.""" if not session: session = get_session() session.add(self) @@ -63,7 +62,7 @@ class NovaBase(object): raise def delete(self, session=None): - """Delete this object""" + """Delete this object.""" self.deleted = True self.deleted_at = datetime.datetime.utcnow() self.save(session=session) @@ -141,7 +140,8 @@ class NovaBase(object): class Service(BASE, NovaBase): - """Represents a running service on a host""" + """Represents a running service on a host.""" + __tablename__ = 'services' id = Column(Integer, primary_key=True) host = Column(String(255)) # , ForeignKey('hosts.id')) @@ -152,7 +152,7 @@ class Service(BASE, NovaBase): class Instance(BASE, NovaBase): - """Represents a guest vm""" + """Represents a guest vm.""" __tablename__ = 'instances' id = Column(Integer, primary_key=True) internal_id = Column(Integer, unique=True) @@ -228,7 +228,7 @@ class Instance(BASE, NovaBase): class Volume(BASE, NovaBase): - """Represents a block storage device that can be attached to a vm""" + """Represents a block storage device that can be attached to a vm.""" __tablename__ = 'volumes' id = Column(Integer, primary_key=True) ec2_id = Column(String(12), unique=True) @@ -263,7 +263,7 @@ class Volume(BASE, NovaBase): class Quota(BASE, NovaBase): - """Represents quota overrides for a project""" + """Represents quota overrides for a project.""" __tablename__ = 'quotas' id = Column(Integer, primary_key=True) @@ -277,7 +277,7 @@ class Quota(BASE, NovaBase): class ExportDevice(BASE, NovaBase): - """Represates a shelf and blade that a volume can be exported on""" + """Represates a shelf and blade that a volume can be exported on.""" __tablename__ = 'export_devices' __table_args__ = (schema.UniqueConstraint("shelf_id", "blade_id"), {'mysql_engine': 'InnoDB'}) @@ -316,7 +316,7 @@ class SecurityGroupInstanceAssociation(BASE, NovaBase): class SecurityGroup(BASE, NovaBase): - """Represents a security group""" + """Represents a security group.""" __tablename__ = 'security_groups' id = Column(Integer, primary_key=True) @@ -346,7 +346,7 @@ class SecurityGroup(BASE, NovaBase): class SecurityGroupIngressRule(BASE, NovaBase): - """Represents a rule in a security group""" + """Represents a rule in a security group.""" __tablename__ = 'security_group_rules' id = Column(Integer, primary_key=True) @@ -368,7 +368,7 @@ class SecurityGroupIngressRule(BASE, NovaBase): class KeyPair(BASE, NovaBase): - """Represents a public key pair for ssh""" + """Represents a public key pair for ssh.""" __tablename__ = 'key_pairs' id = Column(Integer, primary_key=True) @@ -381,7 +381,7 @@ class KeyPair(BASE, NovaBase): class Network(BASE, NovaBase): - """Represents a network""" + """Represents a network.""" __tablename__ = 'networks' __table_args__ = (schema.UniqueConstraint("vpn_public_address", "vpn_public_port"), @@ -410,9 +410,12 @@ class Network(BASE, NovaBase): class AuthToken(BASE, NovaBase): - """Represents an authorization token for all API transactions. Fields - are a string representing the actual token and a user id for mapping - to the actual user""" + """Represents an authorization token for all API transactions. + + Fields are a string representing the actual token and a user id for + mapping to the actual user + + """ __tablename__ = 'auth_tokens' token_hash = Column(String(255), primary_key=True) user_id = Column(Integer) @@ -423,7 +426,7 @@ class AuthToken(BASE, NovaBase): # TODO(vish): can these both come from the same baseclass? class FixedIp(BASE, NovaBase): - """Represents a fixed ip for an instance""" + """Represents a fixed ip for an instance.""" __tablename__ = 'fixed_ips' id = Column(Integer, primary_key=True) address = Column(String(255)) @@ -442,7 +445,7 @@ class FixedIp(BASE, NovaBase): class User(BASE, NovaBase): - """Represents a user""" + """Represents a user.""" __tablename__ = 'users' id = Column(String(255), primary_key=True) @@ -454,7 +457,7 @@ class User(BASE, NovaBase): class Project(BASE, NovaBase): - """Represents a project""" + """Represents a project.""" __tablename__ = 'projects' id = Column(String(255), primary_key=True) name = Column(String(255)) @@ -502,7 +505,7 @@ class UserProjectAssociation(BASE, NovaBase): class FloatingIp(BASE, NovaBase): - """Represents a floating ip that dynamically forwards to a fixed ip""" + """Represents a floating ip that dynamically forwards to a fixed ip.""" __tablename__ = 'floating_ips' id = Column(Integer, primary_key=True) address = Column(String(255)) @@ -518,7 +521,11 @@ class FloatingIp(BASE, NovaBase): def register_models(): - """Register Models and create metadata""" + """Register Models and create metadata. + + Called from nova.db.sqlalchemy.__init__ as part of loading the driver, + it will never need to be called explicitly elsewhere. + """ from sqlalchemy import create_engine models = (Service, Instance, Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, diff --git a/nova/image/service.py b/nova/image/service.py index 37cadddccdc9..52ddd4e0f9de 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -45,13 +45,9 @@ class BaseImageService(object): Returns a sequence of mappings of id and name information about images. - :retval a sequence of mappings with the following signature: - - [ - {'id': opaque id of image, - 'name': name of image - }, ... - ] + :rtype: array + :retval: a sequence of mappings with the following signature + {'id': opaque id of image, 'name': name of image} """ raise NotImplementedError @@ -60,19 +56,17 @@ class BaseImageService(object): """ Returns a sequence of mappings of detailed information about images. - :retval a sequence of mappings with the following signature: - - [ - {'id': opaque id of image, - 'name': name of image, - 'created_at': creation timestamp, - 'updated_at': modification timestamp, - 'deleted_at': deletion timestamp or None, - 'deleted': boolean indicating if image has been deleted, - 'status': string description of image status, - 'is_public': boolean indicating if image is public - }, ... - ] + :rtype: array + :retval: a sequence of mappings with the following signature + {'id': opaque id of image, + 'name': name of image, + 'created_at': creation timestamp, + 'updated_at': modification timestamp, + 'deleted_at': deletion timestamp or None, + 'deleted': boolean indicating if image has been deleted, + 'status': string description of image status, + 'is_public': boolean indicating if image is public + } If the service does not implement a method that provides a detailed set of information about images, then the method should raise diff --git a/nova/manager.py b/nova/manager.py index 4244b2db4e82..a6efb8732ca2 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -15,8 +15,40 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + """ -Base class for managers of different parts of the system +Managers are responsible for a certain aspect of the sytem. It is a logical +grouping of code relating to a portion of the system. In general other +components should be using the manager to make changes to the components that +it is responsible for. + +For example, other components that need to deal with volumes in some way, +should do so by calling methods on the VolumeManager instead of directly +changing fields in the database. This allows us to keep all of the code +relating to volumes in the same place. + +We have adopted a basic strategy of Smart managers and dumb data, which means +rather than attaching methods to data objects, components should call manager +methods that act on the data. + +Methods on managers that can be executed locally should be called directly. If +a particular method must execute on a remote host, this should be done via rpc +to the service that wraps the manager + +Managers should be responsible for most of the db access, and +non-implementation specific data. Anything implementation specific that can't +be generalized should be done by the Driver. + +In general, we prefer to have one manager with multiple drivers for different +implementations, but sometimes it makes sense to have multiple managers. You +can think of it this way: Abstract different overall strategies at the manager +level(FlatNetwork vs VlanNetwork), and different implementations at the driver +level(LinuxNetDriver vs CiscoNetDriver). + +Managers will often provide methods for initial setup of a host or periodic +tasksto a wrapping service. + +This module provides Manager, a base class for managers. """ from nova import utils diff --git a/nova/network/manager.py b/nova/network/manager.py index 8a20cb491443..b033bb0a4c04 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -17,7 +17,30 @@ # under the License. """ -Network Hosts are responsible for allocating ips and setting up network +Network Hosts are responsible for allocating ips and setting up network. + +There are multiple backend drivers that handle specific types of networking +topologies. All of the network commands are issued to a subclass of +:class:`NetworkManager`. + +**Related Flags** + +:network_driver: Driver to use for network creation +:flat_network_bridge: Bridge device for simple network instances +:flat_network_dns: Dns for simple network +:flat_network_dhcp_start: Dhcp start for FlatDhcp +:vlan_start: First VLAN for private networks +:vpn_ip: Public IP for the cloudpipe VPN servers +:vpn_start: First Vpn port for private networks +:cnt_vpn_clients: Number of addresses reserved for vpn clients +:network_size: Number of addresses in each private subnet +:floating_range: Floating IP address block +:fixed_range: Fixed IP address block +:date_dhcp_on_disassociate: Whether to update dhcp when fixed_ip + is disassociated +:fixed_ip_disassociate_timeout: Seconds after which a deallocated ip + is disassociated + """ import datetime @@ -63,15 +86,16 @@ flags.DEFINE_integer('fixed_ip_disassociate_timeout', 600, class AddressAlreadyAllocated(exception.Error): - """Address was already allocated""" + """Address was already allocated.""" pass class NetworkManager(manager.Manager): - """Implements common network manager functionality + """Implements common network manager functionality. - This class must be subclassed. + This class must be subclassed to support specific topologies. """ + def __init__(self, network_driver=None, *args, **kwargs): if not network_driver: network_driver = FLAGS.network_driver @@ -86,7 +110,7 @@ class NetworkManager(manager.Manager): self._on_set_network_host(ctxt, network['id']) def set_network_host(self, context, network_id): - """Safely sets the host of the network""" + """Safely sets the host of the network.""" logging.debug("setting network host") host = self.db.network_set_host(context, network_id, @@ -95,34 +119,34 @@ class NetworkManager(manager.Manager): return host def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool""" + """Gets a fixed ip from the pool.""" raise NotImplementedError() def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" raise NotImplementedError() def setup_fixed_ip(self, context, address): - """Sets up rules for fixed ip""" + """Sets up rules for fixed ip.""" raise NotImplementedError() def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a network""" + """Called when this host becomes the host for a network.""" raise NotImplementedError() def setup_compute_network(self, context, instance_id): - """Sets up matching network for compute hosts""" + """Sets up matching network for compute hosts.""" raise NotImplementedError() def allocate_floating_ip(self, context, project_id): - """Gets an floating ip from the pool""" + """Gets an floating ip from the pool.""" # TODO(vish): add floating ips through manage command return self.db.floating_ip_allocate_address(context, self.host, project_id) def associate_floating_ip(self, context, floating_address, fixed_address): - """Associates an floating ip to a fixed ip""" + """Associates an floating ip to a fixed ip.""" self.db.floating_ip_fixed_ip_associate(context, floating_address, fixed_address) @@ -130,18 +154,18 @@ class NetworkManager(manager.Manager): self.driver.ensure_floating_forward(floating_address, fixed_address) def disassociate_floating_ip(self, context, floating_address): - """Disassociates a floating ip""" + """Disassociates a floating ip.""" fixed_address = self.db.floating_ip_disassociate(context, floating_address) self.driver.unbind_floating_ip(floating_address) self.driver.remove_floating_forward(floating_address, fixed_address) def deallocate_floating_ip(self, context, floating_address): - """Returns an floating ip to the pool""" + """Returns an floating ip to the pool.""" self.db.floating_ip_deallocate(context, floating_address) def lease_fixed_ip(self, context, mac, address): - """Called by dhcp-bridge when ip is leased""" + """Called by dhcp-bridge when ip is leased.""" logging.debug("Leasing IP %s", address) fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) instance_ref = fixed_ip_ref['instance'] @@ -158,7 +182,7 @@ class NetworkManager(manager.Manager): logging.warn("IP %s leased that was already deallocated", address) def release_fixed_ip(self, context, mac, address): - """Called by dhcp-bridge when ip is released""" + """Called by dhcp-bridge when ip is released.""" logging.debug("Releasing IP %s", address) fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) instance_ref = fixed_ip_ref['instance'] @@ -183,26 +207,26 @@ class NetworkManager(manager.Manager): self.driver.update_dhcp(context, network_ref['id']) def get_network(self, context): - """Get the network for the current context""" + """Get the network for the current context.""" raise NotImplementedError() def create_networks(self, context, num_networks, network_size, *args, **kwargs): - """Create networks based on parameters""" + """Create networks based on parameters.""" raise NotImplementedError() @property def _bottom_reserved_ips(self): # pylint: disable-msg=R0201 - """Number of reserved ips at the bottom of the range""" + """Number of reserved ips at the bottom of the range.""" return 2 # network, gateway @property def _top_reserved_ips(self): # pylint: disable-msg=R0201 - """Number of reserved ips at the top of the range""" + """Number of reserved ips at the top of the range.""" return 1 # broadcast def _create_fixed_ips(self, context, network_id): - """Create all fixed ips for network""" + """Create all fixed ips for network.""" network_ref = self.db.network_get(context, network_id) # NOTE(vish): Should these be properties of the network as opposed # to properties of the manager class? @@ -222,10 +246,10 @@ class NetworkManager(manager.Manager): class FlatManager(NetworkManager): - """Basic network where no vlans are used""" + """Basic network where no vlans are used.""" def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool""" + """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute # with a network, or a cluster of computes with a network # and use that network here with a method like @@ -239,21 +263,21 @@ class FlatManager(NetworkManager): return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" self.db.fixed_ip_update(context, address, {'allocated': False}) self.db.fixed_ip_disassociate(context.elevated(), address) def setup_compute_network(self, context, instance_id): - """Network is created manually""" + """Network is created manually.""" pass def setup_fixed_ip(self, context, address): - """Currently no setup""" + """Currently no setup.""" pass def create_networks(self, context, cidr, num_networks, network_size, *args, **kwargs): - """Create networks based on parameters""" + """Create networks based on parameters.""" fixed_net = IPy.IP(cidr) for index in range(num_networks): start = index * network_size @@ -271,7 +295,7 @@ class FlatManager(NetworkManager): self._create_fixed_ips(context, network_ref['id']) def get_network(self, context): - """Get the network for the current context""" + """Get the network for the current context.""" # NOTE(vish): To support mutilple network hosts, This could randomly # select from multiple networks instead of just # returning the one. It could also potentially be done @@ -280,7 +304,7 @@ class FlatManager(NetworkManager): FLAGS.flat_network_bridge) def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a network""" + """Called when this host becomes the host for a network.""" net = {} net['injected'] = True net['bridge'] = FLAGS.flat_network_bridge @@ -289,19 +313,19 @@ class FlatManager(NetworkManager): class FlatDHCPManager(NetworkManager): - """Flat networking with dhcp""" + """Flat networking with dhcp.""" def setup_fixed_ip(self, context, address): - """Setup dhcp for this network""" + """Setup dhcp for this network.""" network_ref = db.fixed_ip_get_by_address(context, address) self.driver.update_dhcp(context, network_ref['id']) def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" self.db.fixed_ip_update(context, address, {'allocated': False}) def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a project""" + """Called when this host becomes the host for a project.""" super(FlatDHCPManager, self)._on_set_network_host(context, network_id) network_ref = self.db.network_get(context, network_id) self.db.network_update(context, @@ -313,11 +337,11 @@ class FlatDHCPManager(NetworkManager): class VlanManager(NetworkManager): - """Vlan network with dhcp""" + """Vlan network with dhcp.""" @defer.inlineCallbacks def periodic_tasks(self, context=None): - """Tasks to be run at a periodic interval""" + """Tasks to be run at a periodic interval.""" yield super(VlanManager, self).periodic_tasks(context) now = datetime.datetime.utcnow() timeout = FLAGS.fixed_ip_disassociate_timeout @@ -330,13 +354,13 @@ class VlanManager(NetworkManager): def init_host(self): """Do any initialization that needs to be run if this is a - standalone service. + standalone service. """ super(VlanManager, self).init_host() self.driver.init_host() def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): - """Gets a fixed ip from the pool""" + """Gets a fixed ip from the pool.""" # TODO(vish): This should probably be getting project_id from # the instance, but it is another trip to the db. # Perhaps this method should take an instance_ref. @@ -356,11 +380,11 @@ class VlanManager(NetworkManager): return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): - """Returns a fixed ip to the pool""" + """Returns a fixed ip to the pool.""" self.db.fixed_ip_update(context, address, {'allocated': False}) def setup_fixed_ip(self, context, address): - """Sets forwarding rules and dhcp for fixed ip""" + """Sets forwarding rules and dhcp for fixed ip.""" fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) network_ref = self.db.fixed_ip_get_network(context, address) if self.db.instance_is_vpn(context, fixed_ip_ref['instance_id']): @@ -370,19 +394,19 @@ class VlanManager(NetworkManager): self.driver.update_dhcp(context, network_ref['id']) def setup_compute_network(self, context, instance_id): - """Sets up matching network for compute hosts""" + """Sets up matching network for compute hosts.""" network_ref = db.network_get_by_instance(context, instance_id) self.driver.ensure_vlan_bridge(network_ref['vlan'], network_ref['bridge']) def restart_nets(self): - """Ensure the network for each user is enabled""" + """Ensure the network for each user is enabled.""" # TODO(vish): Implement this pass def create_networks(self, context, cidr, num_networks, network_size, vlan_start, vpn_start): - """Create networks based on parameters""" + """Create networks based on parameters.""" fixed_net = IPy.IP(cidr) for index in range(num_networks): vlan = vlan_start + index @@ -407,12 +431,12 @@ class VlanManager(NetworkManager): self._create_fixed_ips(context, network_ref['id']) def get_network(self, context): - """Get the network for the current context""" + """Get the network for the current context.""" return self.db.project_get_network(context.elevated(), context.project_id) def _on_set_network_host(self, context, network_id): - """Called when this host becomes the host for a network""" + """Called when this host becomes the host for a network.""" network_ref = self.db.network_get(context, network_id) net = {} net['vpn_public_address'] = FLAGS.vpn_ip @@ -424,11 +448,11 @@ class VlanManager(NetworkManager): @property def _bottom_reserved_ips(self): - """Number of reserved ips at the bottom of the range""" + """Number of reserved ips at the bottom of the range.""" return super(VlanManager, self)._bottom_reserved_ips + 1 # vpn server @property def _top_reserved_ips(self): - """Number of reserved ips at the top of the range""" + """Number of reserved ips at the top of the range.""" parent_reserved = super(VlanManager, self)._top_reserved_ips return parent_reserved + FLAGS.cnt_vpn_clients diff --git a/nova/service.py b/nova/service.py index d53d92b65494..9454d4049bd7 100644 --- a/nova/service.py +++ b/nova/service.py @@ -17,7 +17,12 @@ # under the License. """ -Generic Node baseclass for all workers that run on hosts +A service is a very thin wrapper around a Manager object. It exposes the +manager's public methods to other components of the system via rpc. It will +report state periodically to the database and is responsible for initiating +any periodic tasts that need to be executed on a given host. + +This module contains Service, a generic baseclass for all workers. """ import inspect diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 501162465383..7376a11dd0cf 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -24,6 +24,7 @@ flags.DECLARE('volume_driver', 'nova.volume.manager') FLAGS.volume_driver = 'nova.volume.driver.FakeISCSIDriver' FLAGS.connection_type = 'fake' FLAGS.fake_rabbit = True +flags.DECLARE('auth_driver', 'nova.auth.manager') FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 7ff9b58d7f49..12321a96fbfb 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -16,7 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. """ -Tests for Volume Code +Tests for Volume Code. + """ import logging @@ -33,7 +34,8 @@ FLAGS = flags.FLAGS class VolumeTestCase(test.TrialTestCase): - """Test Case for volumes""" + """Test Case for volumes.""" + def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() @@ -44,7 +46,7 @@ class VolumeTestCase(test.TrialTestCase): @staticmethod def _create_volume(size='0'): - """Create a volume object""" + """Create a volume object.""" vol = {} vol['size'] = size vol['user_id'] = 'fake' @@ -56,7 +58,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_create_delete_volume(self): - """Test volume can be created and deleted""" + """Test volume can be created and deleted.""" volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) self.assertEqual(volume_id, db.volume_get(context.get_admin_context(), @@ -70,7 +72,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_big_volume(self): - """Ensure failure if a too large of a volume is requested""" + """Ensure failure if a too large of a volume is requested.""" # FIXME(vish): validation needs to move into the data layer in # volume_create defer.returnValue(True) @@ -83,7 +85,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_many_volumes(self): - """Ensure that NoMoreTargets is raised when we run out of volumes""" + """Ensure that NoMoreTargets is raised when we run out of volumes.""" vols = [] total_slots = FLAGS.iscsi_num_targets for _index in xrange(total_slots): @@ -100,7 +102,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_attach_detach_volume(self): - """Make sure volume can be attached and detached from instance""" + """Make sure volume can be attached and detached from instance.""" inst = {} inst['image_id'] = 'ami-test' inst['reservation_id'] = 'r-fakeres' @@ -149,12 +151,12 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_concurrent_volumes_get_different_targets(self): - """Ensure multiple concurrent volumes get different targets""" + """Ensure multiple concurrent volumes get different targets.""" volume_ids = [] targets = [] def _check(volume_id): - """Make sure targets aren't duplicated""" + """Make sure targets aren't duplicated.""" volume_ids.append(volume_id) admin_context = context.get_admin_context() iscsi_target = db.volume_get_iscsi_target_num(admin_context, diff --git a/nova/utils.py b/nova/utils.py index e7892a2125d2..2970b93bbc61 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -21,6 +21,7 @@ System-level utilities and helper functions. """ import datetime +import functools import inspect import logging import os diff --git a/nova/virt/connection.py b/nova/virt/connection.py index 34e37adf7587..11f0fa8ced65 100644 --- a/nova/virt/connection.py +++ b/nova/virt/connection.py @@ -17,7 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Abstraction of the underlying virtualization API""" +"""Abstraction of the underlying virtualization API.""" import logging import sys @@ -32,13 +32,26 @@ FLAGS = flags.FLAGS def get_connection(read_only=False): - """Returns an object representing the connection to a virtualization - platform. This could be nova.virt.fake.FakeConnection in test mode, - a connection to KVM or QEMU via libvirt, or a connection to XenServer - or Xen Cloud Platform via XenAPI. + """ + Returns an object representing the connection to a virtualization + platform. + + This could be :mod:`nova.virt.fake.FakeConnection` in test mode, + a connection to KVM, QEMU, or UML via :mod:`libvirt_conn`, or a connection + to XenServer or Xen Cloud Platform via :mod:`xenapi`. Any object returned here must conform to the interface documented by - FakeConnection. + :mod:`FakeConnection`. + + **Related flags** + + :connection_type: A string literal that falls through a if/elif structure + to determine what virtualization mechanism to use. + Values may be + + * fake + * libvirt + * xenapi """ # TODO(termie): maybe lazy load after initial check for permissions # TODO(termie): check whether we can be disconnected diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 66eff4c665ea..f855523d3634 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -18,8 +18,11 @@ # under the License. """ -A fake (in-memory) hypervisor+api. Allows nova testing w/o a hypervisor. -This module also documents the semantics of real hypervisor connections. +A fake (in-memory) hypervisor+api. + +Allows nova testing w/o a hypervisor. This module also documents the +semantics of real hypervisor connections. + """ from twisted.internet import defer diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 4f7fd72f0cd9..535a3b53ec21 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -18,7 +18,27 @@ # under the License. """ -A connection to a hypervisor (e.g. KVM) through libvirt. +A connection to a hypervisor through libvirt. + +Supports KVM, QEMU, UML, and XEN. + +**Related Flags** + +:libvirt_type: Libvirt domain type. Can be kvm, qemu, uml, xen + (default: kvm). +:libvirt_uri: Override for the default libvirt URI (depends on libvirt_type). +:libvirt_xml_template: Libvirt XML Template (QEmu/KVM). +:libvirt_xen_xml_template: Libvirt XML Template (Xen). +:libvirt_uml_xml_template: Libvirt XML Template (User Mode Linux). +:libvirt_rescue_xml_template: XML template for rescue mode (KVM & QEMU). +:libvirt_rescue_xen_xml_template: XML templage for rescue mode (XEN). +:libvirt_rescue_uml_xml_template: XML template for rescue mode (UML). +:rescue_image_id: Rescue ami image (default: ami-rescue). +:rescue_kernel_id: Rescue aki image (default: aki-rescue). +:rescue_ramdisk_id: Rescue ari image (default: ari-rescue). +:injected_network_template: Template file for injected network +:allow_project_net_traffic: Whether to allow in project network traffic + """ import logging diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index a17e405ab1ed..0f563aa41c12 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -33,6 +33,18 @@ long-running operations. FIXME: get_info currently doesn't conform to these rules, and will block the reactor thread if the VM.get_by_name_label or VM.get_record calls block. + +**Related Flags** + +:xenapi_connection_url: URL for connection to XenServer/Xen Cloud Platform. +:xenapi_connection_username: Username for connection to XenServer/Xen Cloud + Platform (default: root). +:xenapi_connection_password: Password for connection to XenServer/Xen Cloud + Platform. +:xenapi_task_poll_interval: The interval (seconds) used for polling of + remote tasks (Async.VM.start, etc) + (default: 0.5). + """ import logging diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 6b05107049eb..156aad2a0954 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -15,9 +15,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ -Drivers for volumes +Drivers for volumes. + """ import logging @@ -54,7 +54,7 @@ flags.DEFINE_string('iscsi_ip_prefix', '127.0', class VolumeDriver(object): - """Executes commands relating to Volumes""" + """Executes commands relating to Volumes.""" def __init__(self, execute=process.simple_execute, sync_exec=utils.execute, *args, **kwargs): # NOTE(vish): db is set by Manager @@ -88,7 +88,7 @@ class VolumeDriver(object): @defer.inlineCallbacks def create_volume(self, volume): - """Creates a logical volume""" + """Creates a logical volume.""" if int(volume['size']) == 0: sizestr = '100M' else: @@ -100,7 +100,7 @@ class VolumeDriver(object): @defer.inlineCallbacks def delete_volume(self, volume): - """Deletes a logical volume""" + """Deletes a logical volume.""" yield self._try_execute("sudo lvremove -f %s/%s" % (FLAGS.volume_group, volume['name'])) @@ -114,39 +114,39 @@ class VolumeDriver(object): escaped_name)) def ensure_export(self, context, volume): - """Safely and synchronously recreates an export for a logical volume""" + """Synchronously recreates an export for a logical volume.""" raise NotImplementedError() @defer.inlineCallbacks def create_export(self, context, volume): - """Exports the volume""" + """Exports the volume.""" raise NotImplementedError() @defer.inlineCallbacks def remove_export(self, context, volume): - """Removes an export for a logical volume""" + """Removes an export for a logical volume.""" raise NotImplementedError() @defer.inlineCallbacks def discover_volume(self, volume): - """Discover volume on a remote host""" + """Discover volume on a remote host.""" raise NotImplementedError() @defer.inlineCallbacks def undiscover_volume(self, volume): - """Undiscover volume on a remote host""" + """Undiscover volume on a remote host.""" raise NotImplementedError() class AOEDriver(VolumeDriver): - """Implements AOE specific volume commands""" + """Implements AOE specific volume commands.""" def ensure_export(self, context, volume): # NOTE(vish): we depend on vblade-persist for recreating exports pass def _ensure_blades(self, context): - """Ensure that blades have been created in datastore""" + """Ensure that blades have been created in datastore.""" total_blades = FLAGS.num_shelves * FLAGS.blades_per_shelf if self.db.export_device_count(context) >= total_blades: return @@ -157,7 +157,7 @@ class AOEDriver(VolumeDriver): @defer.inlineCallbacks def create_export(self, context, volume): - """Creates an export for a logical volume""" + """Creates an export for a logical volume.""" self._ensure_blades(context) (shelf_id, blade_id) = self.db.volume_allocate_shelf_and_blade(context, @@ -184,7 +184,7 @@ class AOEDriver(VolumeDriver): @defer.inlineCallbacks def remove_export(self, context, volume): - """Removes an export for a logical volume""" + """Removes an export for a logical volume.""" (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume['id']) @@ -195,39 +195,40 @@ class AOEDriver(VolumeDriver): @defer.inlineCallbacks def discover_volume(self, _volume): - """Discover volume on a remote host""" + """Discover volume on a remote host.""" yield self._execute("sudo aoe-discover") yield self._execute("sudo aoe-stat", check_exit_code=False) @defer.inlineCallbacks def undiscover_volume(self, _volume): - """Undiscover volume on a remote host""" + """Undiscover volume on a remote host.""" yield class FakeAOEDriver(AOEDriver): - """Logs calls instead of executing""" + """Logs calls instead of executing.""" + def __init__(self, *args, **kwargs): super(FakeAOEDriver, self).__init__(execute=self.fake_execute, sync_exec=self.fake_execute, *args, **kwargs) def check_for_setup_error(self): - """Returns an error if prerequisites aren't met""" + """No setup necessary in fake mode.""" pass @staticmethod def fake_execute(cmd, *_args, **_kwargs): - """Execute that simply logs the command""" + """Execute that simply logs the command.""" logging.debug("FAKE AOE: %s", cmd) return (None, None) class ISCSIDriver(VolumeDriver): - """Executes commands relating to ISCSI volumes""" + """Executes commands relating to ISCSI volumes.""" def ensure_export(self, context, volume): - """Safely and synchronously recreates an export for a logical volume""" + """Synchronously recreates an export for a logical volume.""" iscsi_target = self.db.volume_get_iscsi_target_num(context, volume['id']) iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) @@ -242,7 +243,7 @@ class ISCSIDriver(VolumeDriver): check_exit_code=False) def _ensure_iscsi_targets(self, context, host): - """Ensure that target ids have been created in datastore""" + """Ensure that target ids have been created in datastore.""" host_iscsi_targets = self.db.iscsi_target_count_by_host(context, host) if host_iscsi_targets >= FLAGS.iscsi_num_targets: return @@ -253,7 +254,7 @@ class ISCSIDriver(VolumeDriver): @defer.inlineCallbacks def create_export(self, context, volume): - """Creates an export for a logical volume""" + """Creates an export for a logical volume.""" self._ensure_iscsi_targets(context, volume['host']) iscsi_target = self.db.volume_allocate_iscsi_target(context, volume['id'], @@ -269,7 +270,7 @@ class ISCSIDriver(VolumeDriver): @defer.inlineCallbacks def remove_export(self, context, volume): - """Removes an export for a logical volume""" + """Removes an export for a logical volume.""" iscsi_target = self.db.volume_get_iscsi_target_num(context, volume['id']) yield self._execute("sudo ietadm --op delete --tid=%s " @@ -279,6 +280,7 @@ class ISCSIDriver(VolumeDriver): @defer.inlineCallbacks def _get_name_and_portal(self, volume_name, host): + """Gets iscsi name and portal from volume name and host.""" (out, _err) = yield self._execute("sudo iscsiadm -m discovery -t " "sendtargets -p %s" % host) for target in out.splitlines(): @@ -290,7 +292,7 @@ class ISCSIDriver(VolumeDriver): @defer.inlineCallbacks def discover_volume(self, volume): - """Discover volume on a remote host""" + """Discover volume on a remote host.""" (iscsi_name, iscsi_portal) = yield self._get_name_and_portal(volume['name'], volume['host']) @@ -303,7 +305,7 @@ class ISCSIDriver(VolumeDriver): @defer.inlineCallbacks def undiscover_volume(self, volume): - """Undiscover volume on a remote host""" + """Undiscover volume on a remote host.""" (iscsi_name, iscsi_portal) = yield self._get_name_and_portal(volume['name'], volume['host']) @@ -317,18 +319,18 @@ class ISCSIDriver(VolumeDriver): class FakeISCSIDriver(ISCSIDriver): - """Logs calls instead of executing""" + """Logs calls instead of executing.""" def __init__(self, *args, **kwargs): super(FakeISCSIDriver, self).__init__(execute=self.fake_execute, sync_exec=self.fake_execute, *args, **kwargs) def check_for_setup_error(self): - """Returns an error if prerequisites aren't met""" + """No setup necessary in fake mode.""" pass @staticmethod def fake_execute(cmd, *_args, **_kwargs): - """Execute that simply logs the command""" + """Execute that simply logs the command.""" logging.debug("FAKE ISCSI: %s", cmd) return (None, None) diff --git a/nova/volume/manager.py b/nova/volume/manager.py index ee1c019ad9f9..589e7d7d9752 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -15,10 +15,31 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """ -Volume manager manages creating, attaching, detaching, and -destroying persistent storage volumes, ala EBS. +Volume manager manages creating, attaching, detaching, and persistent storage. + +Persistant storage volumes keep their state independent of instances. You can +attach to an instance, terminate the instance, spawn a new instance (even +one from a different image) and re-attach the volume with the same data +intact. + +**Related Flags** + +:volume_topic: What :mod:`rpc` topic to listen to (default: `volume`). +:volume_manager: The module name of a class derived from + :class:`manager.Manager` (default: + :class:`nova.volume.manager.AOEManager`). +:storage_availability_zone: Defaults to `nova`. +:volume_driver: Used by :class:`AOEManager`. Defaults to + :class:`nova.volume.driver.AOEDriver`. +:num_shelves: Number of shelves for AoE (default: 100). +:num_blades: Number of vblades per shelf to allocate AoE storage from + (default: 16). +:volume_group: Name of the group that will contain exported volumes (default: + `nova-volumes`) +:aoe_eth_dev: Device name the volumes will be exported on (default: `eth0`). +:num_shell_tries: Number of times to attempt to run AoE commands (default: 3) + """ import logging @@ -44,8 +65,9 @@ flags.DEFINE_boolean('use_local_volumes', True, class VolumeManager(manager.Manager): - """Manages attachable block storage devices""" + """Manages attachable block storage devices.""" def __init__(self, volume_driver=None, *args, **kwargs): + """Load the driver from the one specified in args, or from flags.""" if not volume_driver: volume_driver = FLAGS.volume_driver self.driver = utils.import_object(volume_driver) @@ -56,8 +78,7 @@ class VolumeManager(manager.Manager): def init_host(self): """Do any initialization that needs to be run if this is a - standalone service. - """ + standalone service.""" self.driver.check_for_setup_error() ctxt = context.get_admin_context() volumes = self.db.volume_get_all_by_host(ctxt, self.host) @@ -67,7 +88,7 @@ class VolumeManager(manager.Manager): @defer.inlineCallbacks def create_volume(self, context, volume_id): - """Creates and exports the volume""" + """Creates and exports the volume.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) logging.info("volume %s: creating", volume_ref['name']) @@ -95,7 +116,7 @@ class VolumeManager(manager.Manager): @defer.inlineCallbacks def delete_volume(self, context, volume_id): - """Deletes and unexports volume""" + """Deletes and unexports volume.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['attach_status'] == "attached": @@ -112,10 +133,9 @@ class VolumeManager(manager.Manager): @defer.inlineCallbacks def setup_compute_volume(self, context, volume_id): - """Setup remote volume on compute host + """Setup remote volume on compute host. - Returns path to device. - """ + Returns path to device.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host and FLAGS.use_local_volumes: @@ -126,7 +146,7 @@ class VolumeManager(manager.Manager): @defer.inlineCallbacks def remove_compute_volume(self, context, volume_id): - """Remove remote volume on compute host """ + """Remove remote volume on compute host.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host and FLAGS.use_local_volumes: