Merge branch 'master' into simplenet

README

@@ -15,3 +15,6 @@ To disect it in detail: visit http://github.com/nova/cc
 To taunt it with its weaknesses: use http://github.com/nova/cc/issues
 
 To hack at it: read HACKING
+
+To watch it: http://184.106.205.70:8010
+ and again and again (This is buildbot again)

bin/nova-manage

@@ -220,7 +220,7 @@ def methods_of(obj):
 
 
 if __name__ == '__main__':
-    utils.default_flagfile()
+    utils.default_flagfile('/etc/nova/nova-manage.conf')
     argv = FLAGS(sys.argv)
     script_name = argv.pop(0)
     if len(argv) < 1:

debian/changelog

@@ -1,3 +1,54 @@
+nova (0.2.3-1) UNRELEASED; urgency=low
+
+  * Relax the Twisted dependency to python-twisted-core (rather than the
+    full stack).
+  * Move nova related configuration files into /etc/nova/.
+  * Add a dependency on nginx from nova-objectsstore and install a
+    suitable configuration file.
+  * Ship the CA directory in nova-common.
+  * Add a default flag file for nova-manage to help it find the CA.
+  * If set, pass KernelId and RamdiskId from RunInstances call to the
+    target compute node.
+  * Added --network_path setting to nova-compute's flagfile.
+  * Move templates from python directories to /usr/share/nova.
+  * Add debian/nova-common.dirs to create
+    var/lib/nova/{buckets,CA,images,instances,keys,networks}
+  * Don't pass --daemonize=1 to nova-compute. It's already daemonising
+    by default.
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 14 Jul 2010 12:00:00 -0700
+
+nova (0.2.2-10) UNRELEASED; urgency=low
+
+  * Fixed extra space in vblade-persist
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 13 Jul 2010 19:00:00 -0700
+
+nova (0.2.2-9) UNRELEASED; urgency=low
+
+  * Fixed invalid dn bug in ldap for adding roles
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 12 Jul 2010 15:20:00 -0700
+
+nova (0.2.2-8) UNRELEASED; urgency=low
+
+  * Added a missing comma
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 08 Jul 2010 10:05:00 -0700
+
+nova (0.2.2-7) UNRELEASED; urgency=low
+
+  * Missing files from twisted patch
+  * License upedates
+  * Reformatting/cleanup
+  * Users/ldap bugfixes
+  * Merge fixes
+  * Documentation updates
+  * Vpn key creation fix
+  * Multiple shelves for volumes
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Wed, 07 Jul 2010 18:45:00 -0700
+
 nova (0.2.2-6) UNRELEASED; urgency=low
 
   * Fix to make Key Injection work again

debian/control

@@ -3,36 +3,112 @@ Section: net
 Priority: extra
 Maintainer: Jesse Andrews <jesse@ansolabs.com>
 Build-Depends: debhelper (>= 7)
-Build-Depends-Indep: python-support
+Build-Depends-Indep: python-support, python-setuptools
 Standards-Version: 3.8.4
 XS-Python-Version: 2.6
 
 Package: nova-common
 Architecture: all
-Depends: ${python:Depends}, aoetools, vlan, python-ipy, python-boto, python-m2crypto, python-pycurl, python-twisted (>= 10.0.0-2ubuntu2nebula1), python-daemon, python-redis, python-carrot, python-lockfile, python-gflags, python-tornado, ${misc:Depends}
+Depends: ${python:Depends}, aoetools, vlan, python-ipy, python-boto, python-m2crypto, python-pycurl, python-twisted-core, python-daemon, python-redis, python-carrot, python-lockfile, python-gflags, python-tornado, ${misc:Depends}
 Provides: ${python:Provides}
-Conflicts: nova
-Description: Nova is a cloud
+Description: Nova Cloud Computing - common files
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package contains things that are needed by all parts of Nova.
 
 Package: nova-compute
 Architecture: all
-Depends: nova-common (= ${binary:Version}), kpartx, kvm, python-libvirt, libvirt-bin (>= 0.8.1), ${python:Depends}, ${misc:Depends}
-Description: Nova compute
+Depends: nova-common (= ${binary:Version}), kpartx, kvm, python-libvirt, libvirt-bin (>= 0.7.5), curl, ${python:Depends}, ${misc:Depends}
+Description: Nova Cloud Computing - compute node
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on the nodes that will run your
+ virtual machines.
 
 Package: nova-volume
 Architecture: all
 Depends: nova-common (= ${binary:Version}), vblade, vblade-persist, ${python:Depends}, ${misc:Depends}
-Description: Nova volume
+Description: Nova Cloud Computing - storage
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on your storage nodes.
 
 Package: nova-api
 Architecture: all
 Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends}
-Description: Nova api
+Description: Nova Cloud Computing - API frontend 
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package provides the API frontend.
 
 Package: nova-objectstore
 Architecture: all
-Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends}
-Description: Nova object store
+Depends: nova-common (= ${binary:Version}), nginx, ${python:Depends}, ${misc:Depends}
+Description: Nova Cloud Computing - object store
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on the nodes that will contain your
+ object store.
 
 Package: nova-instancemonitor
 Architecture: all
@@ -42,4 +118,19 @@ Description: Nova instance monitor
 Package: nova-tools
 Architecture: all
 Depends: python-boto, ${python:Depends}, ${misc:Depends}
-Description: CLI tools to access nova
+Description: Nova Cloud Computing - management tools
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package contains admin tools for Nova.

debian/nova-api.conf

@@ -0,0 +1,5 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--fake_users=1
+--datastore_path=/var/lib/nova/keeper

debian/nova-api.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-api
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-api.conf"
 PIDFILE=/var/run/nova-api.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-api; then
   . /etc/default/nova-api

debian/nova-api.install

@@ -1 +1,2 @@
 bin/nova-api usr/bin
+debian/nova-api.conf etc/nova

debian/nova-common.dirs

@@ -0,0 +1,11 @@
+etc/nova
+var/lib/nova/buckets
+var/lib/nova/CA
+var/lib/nova/CA/INTER
+var/lib/nova/CA/newcerts
+var/lib/nova/CA/private
+var/lib/nova/CA/reqs
+var/lib/nova/images
+var/lib/nova/instances
+var/lib/nova/keys
+var/lib/nova/networks

debian/nova-common.install

@@ -1,5 +1,9 @@
 bin/nova-manage usr/bin
-nova/auth/novarc.template             usr/lib/pymodules/python2.6/nova/auth
-nova/cloudpipe/client.ovpn.template   usr/lib/pymodules/python2.6/nova/cloudpipe
-nova/compute/libvirt.xml.template     usr/lib/pymodules/python2.6/nova/compute
+debian/nova-manage.conf etc/nova
+nova/auth/novarc.template             usr/share/nova
+nova/cloudpipe/client.ovpn.template   usr/share/nova
+nova/compute/libvirt.xml.template     usr/share/nova
 usr/lib/python*/*-packages/nova/*
+CA/openssl.cnf.tmpl                   var/lib/nova/CA
+CA/geninter.sh                        var/lib/nova/CA
+CA/genrootca.sh                       var/lib/nova/CA

debian/nova-compute.conf

@@ -0,0 +1,9 @@
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--instances_path=/var/lib/nova/instances
+--networks_path=/var/lib/nova/networks
+--libvirt_xml_template=/usr/share/nova/libvirt.xml.template
+--vpn_client_template=/usr/share/nova/client.ovpn.template
+--credentials_template=/usr/share/nova/novarc.template
+--fake_users=1

debian/nova-compute.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-compute
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-compute.conf"
 PIDFILE=/var/run/nova-compute.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-compute; then
   . /etc/default/nova-compute

debian/nova-compute.install

@@ -1 +1,2 @@
 bin/nova-compute usr/bin
+debian/nova-compute.conf etc/nova

debian/nova-manage.conf

@@ -0,0 +1,4 @@
+--ca_path=/var/lib/nova/CA
+--credentials_template=/usr/share/nova/novarc.template
+--keys_path=/var/lib/nova/keys
+--vpn_client_template=/usr/share/nova/client.ovpn.template

debian/nova-objectstore.conf

@@ -0,0 +1,7 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--fake_users=1
+--images_path=/var/lib/nova/images
+--buckets_path=/var/lib/nova/buckets

debian/nova-objectstore.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-objectstore
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-objectstore.conf"
 PIDFILE=/var/run/nova-objectstore.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-objectstore; then
   . /etc/default/nova-objectstore

debian/nova-objectstore.install

@@ -1 +1,3 @@
 bin/nova-objectstore usr/bin
+debian/nova-objectstore.conf etc/nova
+debian/nova-objectstore.nginx.conf etc/nginx/sites-available

debian/nova-objectstore.links

@@ -0,0 +1 @@
+/etc/nginx/sites-available/nova-objectstore.nginx.conf /etc/nginx/sites-enabled/nova-objectstore.nginx.conf

debian/nova-objectstore.nginx.conf

@@ -0,0 +1,17 @@
+server {
+  listen 3333 default;
+  server_name localhost;
+  client_max_body_size 10m;
+
+  access_log /var/log/nginx/localhost.access.log;
+
+  location ~ /_images/.+ {
+    root /var/lib/nova/images;
+    rewrite ^/_images/(.*)$ /$1 break;
+  }
+
+  location / {
+    proxy_pass http://localhost:3334/;
+  }
+}
+

debian/nova-volume.conf

@@ -0,0 +1,7 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--fake_users=1
+--images_path=/var/lib/nova/images
+--buckets_path=/var/lib/nova/buckets

debian/nova-volume.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-volume
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-volume.conf"
 PIDFILE=/var/run/nova-volume.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-volume; then
   . /etc/default/nova-volume

debian/nova-volume.install

@@ -1 +1,2 @@
 bin/nova-volume usr/bin
+debian/nova-volume.conf etc/nova

nova/auth/users.py

@@ -100,6 +100,10 @@ flags.DEFINE_string('credential_cert_file', 'cert.pem',
                     'Filename of certificate in credentials zip')
 flags.DEFINE_string('credential_rc_file', 'novarc',
                     'Filename of rc in credentials zip')
+flags.DEFINE_string('credential_cert_subject',
+                    '/C=US/ST=California/L=MountainView/O=AnsoLabs/'
+                    'OU=NovaDev/CN=%s-%s',
+                    'Subject for certificate for users')
 flags.DEFINE_string('vpn_ip', '127.0.0.1',
                     'Public IP for the cloudpipe VPN servers')
 
@@ -516,7 +520,7 @@ class UserManager(object):
 
     def __cert_subject(self, uid):
         # FIXME(ja) - this should be pulled from a global configuration
-        return "/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=%s-%s" % (uid, str(datetime.datetime.utcnow().isoformat()))
+        return FLAGS.credential_cert_subject % (uid, utils.isotime())
 
 
 class LDAPWrapper(object):
@@ -706,7 +710,7 @@ class LDAPWrapper(object):
 
     def __create_group(self, group_dn, name, uid,
                        description, member_uids = None):
-        if self.group_exists(name):
+        if self.group_exists(group_dn):
             raise exception.Duplicate("Group can't be created because "
                                       "group %s already exists" % name)
         members = []

nova/endpoint/cloud.py

@@ -115,9 +115,9 @@ class CloudController(object):
 
     def get_metadata(self, ip):
         i = self.get_instance_by_ip(ip)
-        mpi = self._get_mpi_data(i['project_id'])
         if i is None:
             return None
+        mpi = self._get_mpi_data(i['project_id'])
         if i['key_name']:
             keys = {
                 '0': {
@@ -526,6 +526,10 @@ class CloudController(object):
             inst = self.instdir.new()
             # TODO(ja): add ari, aki
             inst['image_id'] = kwargs['image_id']
+            if 'kernel_id' in kwargs:
+                inst['kernel_id'] = kwargs['kernel_id']
+            if 'ramdisk_id' in kwargs:
+                inst['ramdisk_id'] = kwargs['ramdisk_id']
             inst['user_data'] = kwargs.get('user_data', '')
             inst['instance_type'] = kwargs.get('instance_type', 'm1.small')
             inst['reservation_id'] = reservation_id