x/python-ganttclient
Code Issues Proposed changes

merged trunk

Browse Source
This commit is contained in:
Vishvananda Ishaya
2010-07-26 11:06:45 -07:00
parent 385b300bc7 3486677ef1
commit c03883219f
27 changed files with 38 additions and 1460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CA/geninter.sh
View File

@@ -17,7 +17,7 @@
# under the License.
# ARG is the id of the user
export SUBJ=/C=US/ST=California/L=Mountain View/O=Anso Labs/OU=Nova Dev/CN=customer-intCA-$3
export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$1"
mkdir INTER/$1
cd INTER/$1
cp ../../openssl.cnf.tmpl openssl.cnf

14
README
View File

@@ -6,15 +6,19 @@ The Choose Your Own Adventure README for Nova:
To monitor it from a distance: follow @novacc on twitter
To tame it for use in your own cloud: read http://docs.novacc.org/getting.started.html
To tame it for use in your own cloud: read http://nova.openstack.org/getting.started.html
To study its anatomy: read http://docs.novacc.org/architecture.html
To study its anatomy: read http://nova.openstack.org/architecture.html
To disect it in detail: visit http://github.com/nova/cc
To disect it in detail: visit http://code.launchpad.net/nova
To taunt it with its weaknesses: use http://github.com/nova/cc/issues
To taunt it with its weaknesses: use http://bugs.launchpad.net/nova
To watch it: http://hudson.openstack.org
To hack at it: read HACKING
To watch it: http://test.novacc.org/waterfall
To laugh at its PEP8 problems: http://hudson.openstack.org/job/nova-pep8/violations
To cry over its pylint problems: http://hudson.openstack.org/job/nova-pylint/violations

1
docs/.gitignore vendored
View File

@@ -1 +0,0 @@
_build/*

89
docs/Makefile
View File

@@ -1,89 +0,0 @@
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
clean:
-rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/nova.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/nova.qhc"
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
"run these through (pdf)latex."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."

1
docs/_build/.gitignore vendored
View File

@@ -1 +0,0 @@
*

0
docs/_static/.gitignore vendored
View File

0
docs/_templates/.gitignore vendored
View File

48
docs/architecture.rst
View File

@@ -1,48 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
nova System Architecture
========================
Nova is built on a shared-nothing, messaging-based architecture. All of the major nova components can be run on multiple servers. This means that most component to component communication must go via message queue. In order to avoid blocking each component while waiting for a response, we use deferred objects, with a callback that gets triggered when a response is received.
In order to achieve shared-nothing with multiple copies of the same component (especially when the component is an API server that needs to reply with state information in a timely fashion), we need to keep all of our system state in a distributed data system. Updates to system state are written into this system, using atomic transactions when necessary. Requests for state are read out of this system. In limited cases, these read calls are memoized within controllers for short periods of time. (Such a limited case would be, for instance, the current list of system users.)
Components
----------
Below you will find a helpful explanation.
::
[ User Manager ] ---- ( LDAP )
|
| / [ Storage ] - ( ATAoE )
[ API server ] -> [ Cloud ] < AMQP >
| \ [ Nodes ] - ( libvirt/kvm )
< HTTP >
|
[ S3 ]
* API: receives http requests from boto, converts commands to/from API format, and sending requests to cloud controller
* Cloud Controller: global state of system, talks to ldap, s3, and node/storage workers through a queue
* Nodes: worker that spawns instances
* S3: tornado based http/s3 server
* User Manager: create/manage users, which are stored in ldap
* Network Controller: allocate and deallocate IPs and VLANs

215
docs/auth.rst
View File

@@ -1,215 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Auth Documentation
==================
Nova provides RBAC (Role-based access control) of the AWS-type APIs. We define the following roles:
Roles-Based Access Control of AWS-style APIs using SAML Assertions
“Achieving FIPS 199 Moderate certification of a hybrid cloud environment using CloudAudit and declarative C.I.A. classifications”
Introduction
--------------
We will investigate one method for integrating an AWS-style API with US eAuthentication-compatible federated authentication systems, to achieve access controls and limits based on traditional operational roles.
Additionally, we will look at how combining this approach, with an implementation of the CloudAudit APIs, will allow us to achieve a certification under FIPS 199 Moderate classification for a hybrid cloud environment.
Relationship of US eAuth to RBAC
--------------------------------
Typical implementations of US eAuth authentication systems are structured as follows::
[ MS Active Directory or other federated LDAP user store ]
--> backends to…
[ SUN Identity Manager or other SAML Policy Controller ]
--> maps URLs to groups…
[ Apache Policy Agent in front of eAuth-secured Web Application ]
In more ideal implementations, the remainder of the application-specific account information is stored either in extended schema on the LDAP server itself, via the use of a translucent LDAP proxy, or in an independent datastore keyed off of the UID provided via SAML assertion.
Basic AWS API call structure
----------------------------
AWS API calls are traditionally secured via Access and Secret Keys, which are used to sign API calls, along with traditional timestamps to prevent replay attacks. The APIs can be logically grouped into sets that align with five typical roles:
* System User
* System Administrator
* Network Administrator
* Project Manager
* Cloud Administrator
* (IT-Sec?)
There is an additional, conceptual end-user that may or may not have API access:
* (EXTERNAL) End-user / Third-party User
Basic operations are available to any System User:
* Launch Instance
* Terminate Instance (their own)
* Create keypair
* Delete keypair
* Create, Upload, Delete: Buckets and Keys (Object Store) – their own
* Create, Attach, Delete Volume (Block Store) – their own
System Administrators:
* Register/Unregister Machine Image (project-wide)
* Change Machine Image properties (public / private)
* Request / Review CloudAudit Scans
Network Administrator:
* Change Firewall Rules, define Security Groups
* Allocate, Associate, Deassociate Public IP addresses
Project Manager:
* Launch and Terminate Instances (project-wide)
* CRUD of Object and Block store (project-wide)
Cloud Administrator:
* Register / Unregister Kernel and Ramdisk Images
* Register / Unregister Machine Image (any)
Enhancements
------------
* SAML Token passing
* REST interfaces
* SOAP interfaces
Wrapping the SAML token into the API calls.
Then store the UID (fetched via backchannel) into the instance metadata, providing end-to-end auditability of ownership and responsibility, without PII.
CloudAudit APIs
---------------
* Request formats
* Response formats
* Stateless asynchronous queries
CloudAudit queries may spawn long-running processes (similar to launching instances, etc.) They need to return a ReservationId in the same fashion, which can be returned in further queries for updates.
RBAC of CloudAudit API calls is critical, since detailed system information is a system vulnerability.
Type declarations
---------------------
* Data declarations – Volumes and Objects
* System declarations – Instances
Existing API calls to launch instances specific a single, combined “type” flag. We propose to extend this with three additional type declarations, mapping to the “Confidentiality, Integrity, Availability” classifications of FIPS 199. An example API call would look like::
RunInstances type=m1.large number=1 secgroup=default key=mykey confidentiality=low integrity=low availability=low
These additional parameters would also apply to creation of block storage volumes (along with the existing parameter of ‘size’), and creation of object storage ‘buckets’. (C.I.A. classifications on a bucket would be inherited by the keys within this bucket.)
Request Brokering
-----------------
* Cloud Interop
* IMF Registration / PubSub
* Digital C&A
Establishing declarative semantics for individual API calls will allow the cloud environment to seamlessly proxy these API calls to external, third-party vendors – when the requested CIA levels match.
See related work within the Infrastructure 2.0 working group for more information on how the IMF Metadata specification could be utilized to manage registration of these vendors and their C&A credentials.
Dirty Cloud – Hybrid Data Centers
---------------------------------
* CloudAudit bridge interfaces
* Anything in the ARP table
A hybrid cloud environment provides dedicated, potentially co-located physical hardware with a network interconnect to the project or users’ cloud virtual network.
This interconnect is typically a bridged VPN connection. Any machines that can be bridged into a hybrid environment in this fashion (at Layer 2) must implement a minimum version of the CloudAudit spec, such that they can be queried to provide a complete picture of the IT-sec runtime environment.
Network discovery protocols (ARP, CDP) can be applied in this case, and existing protocols (SNMP location data, DNS LOC records) overloaded to provide CloudAudit information.
The Details
-----------
* Preliminary Roles Definitions
* Categorization of available API calls
* SAML assertion vocabulary
System limits
-------------
The following limits need to be defined and enforced:
* Total number of instances allowed (user / project)
* Total number of instances, per instance type (user / project)
* Total number of volumes (user / project)
* Maximum size of volume
* Cumulative size of all volumes
* Total use of object storage (GB)
* Total number of Public IPs
Further Challenges
------------------
* Prioritization of users / jobs in shared computing environments
* Incident response planning
* Limit launch of instances to specific security groups based on AMI
* Store AMIs in LDAP for added property control
The :mod:`rbac` Module
--------------------------
.. automodule:: nova.auth.rbac
:members:
:undoc-members:
:show-inheritance:
The :mod:`signer` Module
------------------------
.. automodule:: nova.auth.signer
:members:
:undoc-members:
:show-inheritance:
The :mod:`users` Module
-----------------------
.. automodule:: nova.auth.users
:members:
:undoc-members:
:show-inheritance:
The :mod:`users_unittest` Module
--------------------------------
.. automodule:: nova.tests.users_unittest
:members:
:undoc-members:
:show-inheritance:
The :mod:`access_unittest` Module
---------------------------------
.. automodule:: nova.tests.access_unittest
:members:
:undoc-members:
:show-inheritance:

31
docs/binaries.rst
View File

@@ -1,31 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Nova Binaries
===============
* nova-api
* nova-compute
* nova-manage
* nova-objectstore
* nova-volume
The configuration of these binaries relies on "flagfiles" using the google
gflags package. If present, the nova.conf file will be used as the flagfile
- otherwise, it must be specified on the command line::
$ python node_worker.py --flagfile flagfile

74
docs/compute.rst
View File

@@ -1,74 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Compute Documentation
=====================
This page contains the Compute Package documentation.
The :mod:`disk` Module
----------------------
.. automodule:: nova.compute.disk
:members:
:undoc-members:
:show-inheritance:
The :mod:`exception` Module
---------------------------
.. automodule:: nova.compute.exception
:members:
:undoc-members:
:show-inheritance:
The :mod:`model` Module
-------------------------
.. automodule:: nova.compute.model
:members:
:undoc-members:
:show-inheritance:
The :mod:`network` Module
-------------------------
.. automodule:: nova.compute.network
:members:
:undoc-members:
:show-inheritance:
The :mod:`node` Module
----------------------
.. automodule:: nova.compute.node
:members:
:undoc-members:
:show-inheritance:
RELATED TESTS
---------------
The :mod:`node_unittest` Module
-------------------------------
.. automodule:: nova.tests.node_unittest
:members:
:undoc-members:
:show-inheritance:

202
docs/conf.py
View File

@@ -1,202 +0,0 @@
# -*- coding: utf-8 -*-
#
# nova documentation build configuration file, created by
# sphinx-quickstart on Sat May 1 15:17:47 2010.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys, os
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(os.path.abspath('/Users/jmckenty/Projects/cc'))
sys.path.append([os.path.abspath('../nova'),os.path.abspath('../'),os.path.abspath('../vendor')])
# -- General configuration -----------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig']
#sphinx_to_github = False
todo_include_todos = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'nova'
copyright = u'2010, United States Government as represented by the Administrator of the National Aeronautics and Space Administration.'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.42'
# The full version, including alpha/beta/rc tags.
release = '0.42'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of documents that shouldn't be included in the build.
#unused_docs = []
# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['_build']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
modindex_common_prefix = ['nova.']
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_use_modindex = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''
# Output file base name for HTML help builder.
htmlhelp_basename = 'novadoc'
# -- Options for LaTeX output --------------------------------------------------
# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'
# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'nova.tex', u'nova Documentation',
u'Anso Labs, LLC', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_use_modindex = True
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

91
docs/endpoint.rst
View File

@@ -1,91 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Endpoint Documentation
======================
This page contains the Endpoint Package documentation.
The :mod:`admin` Module
-----------------------
.. automodule:: nova.endpoint.admin
:members:
:undoc-members:
:show-inheritance:
The :mod:`api` Module
---------------------
.. automodule:: nova.endpoint.api
:members:
:undoc-members:
:show-inheritance:
The :mod:`cloud` Module
-----------------------
.. automodule:: nova.endpoint.cloud
:members:
:undoc-members:
:show-inheritance:
The :mod:`images` Module
------------------------
.. automodule:: nova.endpoint.images
:members:
:undoc-members:
:show-inheritance:
RELATED TESTS
--------------
The :mod:`api_unittest` Module
------------------------------
.. automodule:: nova.tests.api_unittest
:members:
:undoc-members:
:show-inheritance:
The :mod:`api_integration` Module
---------------------------------
.. automodule:: nova.tests.api_integration
:members:
:undoc-members:
:show-inheritance:
The :mod:`cloud_unittest` Module
--------------------------------
.. automodule:: nova.tests.cloud_unittest
:members:
:undoc-members:
:show-inheritance:
The :mod:`network_unittest` Module
----------------------------------
.. automodule:: nova.tests.network_unittest
:members:
:undoc-members:
:show-inheritance:

43
docs/fakes.rst
View File

@@ -1,43 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Nova Fakes
==========
The :mod:`fakevirt` Module
--------------------------
.. automodule:: nova.fakevirt
:members:
:undoc-members:
:show-inheritance:
The :mod:`fakeldap` Module
--------------------------
.. automodule:: nova.auth.fakeldap
:members:
:undoc-members:
:show-inheritance:
The :mod:`fakerabbit` Module
----------------------------
.. automodule:: nova.fakerabbit
:members:
:undoc-members:
:show-inheritance:

148
docs/getting.started.rst
View File

@@ -1,148 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Getting Started with Nova
=========================
GOTTA HAVE A nova.pth file added or it WONT WORK (will write setup.py file soon)
Create a file named nova.pth in your python libraries directory
(usually /usr/local/lib/python2.6/dist-packages) with a single line that points
to the directory where you checked out the source (that contains the nova/
directory).
DEPENDENCIES
------------
Related servers we rely on
* RabbitMQ: messaging queue, used for all communication between components
* OpenLDAP: users, groups (maybe cut)
* ReDIS: Remote Dictionary Store (for fast, shared state data)
* nginx: HTTP server to handle serving large files (because Tornado can't)
Python libraries we don't vendor
* M2Crypto: python library interface for openssl
* curl
Vendored python libaries (don't require any installation)
* Tornado: scalable non blocking web server for api requests
* Twisted: just for the twisted.internet.defer package
* boto: python api for aws api
* IPy: library for managing ip addresses
Recommended
-----------------
* euca2ools: python implementation of aws ec2-tools and ami tools
* build tornado to use C module for evented section
Installation
--------------
::
# system libraries and tools
apt-get install -y aoetools vlan curl
modprobe aoe
# python libraries
apt-get install -y python-setuptools python-dev python-pycurl python-m2crypto
# ON THE CLOUD CONTROLLER
apt-get install -y rabbitmq-server dnsmasq nginx
# build redis from 2.0.0-rc1 source
# setup ldap (slap.sh as root will remove ldap and reinstall it)
NOVA_PATH/nova/auth/slap.sh
/etc/init.d/rabbitmq-server start
# ON VOLUME NODE:
apt-get install -y vblade-persist
# ON THE COMPUTE NODE:
apt-get install -y python-libvirt
apt-get install -y kpartx kvm libvirt-bin
modprobe kvm
# optional packages
apt-get install -y euca2ools
Configuration
---------------
ON CLOUD CONTROLLER
* Add yourself to the libvirtd group, log out, and log back in
* fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances
::
iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773
iptables --table nat --append POSTROUTING --out-interface $PUBLICIFACE -j MASQUERADE
* Configure NginX proxy (/etc/nginx/sites-enabled/default)
::
server {
listen 3333 default;
server-name localhost;
client_max_body_size 10m;
access_log /var/log/nginx/localhost.access.log;
location ~ /_images/.+ {
root NOVA_PATH/images;
rewrite ^/_images/(.*)$ /$1 break;
}
location / {
proxy_pass http://localhost:3334/;
}
}
ON VOLUME NODE
* create a filesystem (you can use an actual disk if you have one spare, default is /dev/sdb)
::
# This creates a 1GB file to create volumes out of
dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10
losetup --show -f MY_FILE_PATH
# replace loop0 below with whatever losetup returns
echo "--storage_dev=/dev/loop0" >> NOVA_PATH/bin/nova.conf
Running
---------
Launch servers
* rabbitmq
* redis
* slapd
* nginx
Launch nova components
* nova-api
* nova-compute
* nova-objectstore
* nova-volume

56
docs/index.rst
View File

@@ -1,56 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Welcome to nova's documentation!
================================
Nova is a cloud computing fabric controller (the main part of an IaaS system) built to match the popular AWS EC2 and S3 APIs.
It is written in Python, using the Tornado and Twisted frameworks, and relies on the standard AMQP messaging protocol,
and the Redis distributed KVS.
Nova is intended to be easy to extend, and adapt. For example, it currently uses
an LDAP server for users and groups, but also includes a fake LDAP server,
that stores data in Redis. It has extensive test coverage, and uses the
Sphinx toolkit (the same as Python itself) for code and user documentation.
While Nova is currently in Beta use within several organizations, the codebase
is very much under active development - there are bugs!
Contents:
.. toctree::
:maxdepth: 2
getting.started
architecture
network
storage
auth
compute
endpoint
nova
fakes
binaries
todo
modules
packages
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

34
docs/modules.rst
View File

@@ -1,34 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Nova Documentation
==================
This page contains the Nova Modules documentation.
Modules:
--------
.. toctree::
:maxdepth: 4
auth
compute
endpoint
fakes
nova
volume

88
docs/network.rst
View File

@@ -1,88 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
nova Networking
================
The nova networking components manage private networks, public IP addressing, VPN connectivity, and firewall rules.
Components
----------
There are several key components:
* NetworkController (Manages address and vlan allocation)
* RoutingNode (NATs public IPs to private IPs, and enforces firewall rules)
* AddressingNode (runs DHCP services for private networks)
* BridgingNode (a subclass of the basic nova ComputeNode)
* TunnelingNode (provides VPN connectivity)
Component Diagram
-----------------
Overview::
(PUBLIC INTERNET)
| \
/ \ / \
[RoutingNode] ... [RN] [TunnelingNode] ... [TN]
| \ / | |
| < AMQP > | |
[AddressingNode]-- (VLAN) ... | (VLAN)... (VLAN) --- [AddressingNode]
\ | \ /
/ \ / \ / \ / \
[BridgingNode] ... [BridgingNode]
[NetworkController] ... [NetworkController]
\ /
< AMQP >
|
/ \
[CloudController]...[CloudController]
While this diagram may not make this entirely clear, nodes and controllers communicate exclusively across the message bus (AMQP, currently).
State Model
-----------
Network State consists of the following facts:
* VLAN assignment (to a project)
* Private Subnet assignment (to a security group) in a VLAN
* Private IP assignments (to running instances)
* Public IP allocations (to a project)
* Public IP associations (to a private IP / running instance)
While copies of this state exist in many places (expressed in IPTables rule chains, DHCP hosts files, etc), the controllers rely only on the distributed "fact engine" for state, queried over RPC (currently AMQP). The NetworkController inserts most records into this datastore (allocating addresses, etc) - however, individual nodes update state e.g. when running instances crash.
The Public Traffic Path
-----------------------
Public Traffic::
(PUBLIC INTERNET)
|
<NAT> <-- [RoutingNode]
|
[AddressingNode] --> |
( VLAN )
| <-- [BridgingNode]
|
<RUNNING INSTANCE>
The RoutingNode is currently implemented using IPTables rules, which implement both NATing of public IP addresses, and the appropriate firewall chains. We are also looking at using Netomata / Clusto to manage NATting within a switch or router, and/or to manage firewall rules within a hardware firewall appliance.
Similarly, the AddressingNode currently manages running DNSMasq instances for DHCP services. However, we could run an internal DHCP server (using Scapy ala Clusto), or even switch to static addressing by inserting the private address into the disk image the same way we insert the SSH keys. (See compute for more details).

91
docs/nova.rst
View File

@@ -1,91 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
NOVA Libraries
===============
The :mod:`crypto` Module
------------------------
.. automodule:: nova.crypto
:members:
:undoc-members:
:show-inheritance:
The :mod:`adminclient` Module
-----------------------------
.. automodule:: nova.adminclient
:members:
:undoc-members:
:show-inheritance:
The :mod:`datastore` Module
---------------------------
.. automodule:: nova.datastore
:members:
:undoc-members:
:show-inheritance:
The :mod:`exception` Module
---------------------------
.. automodule:: nova.exception
:members:
:undoc-members:
:show-inheritance:
The :mod:`flags` Module
---------------------------
.. automodule:: nova.flags
:members:
:undoc-members:
:show-inheritance:
The :mod:`rpc` Module
---------------------------
.. automodule:: nova.rpc
:members:
:undoc-members:
:show-inheritance:
The :mod:`server` Module
---------------------------
.. automodule:: nova.server
:members:
:undoc-members:
:show-inheritance:
The :mod:`test` Module
---------------------------
.. automodule:: nova.test
:members:
:undoc-members:
:show-inheritance:
The :mod:`utils` Module
---------------------------
.. automodule:: nova.utils
:members:
:undoc-members:
:show-inheritance:

66
docs/objectstore.rst
View File

@@ -1,66 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Objectstore Documentation
=========================
This page contains the Objectstore Package documentation.
The :mod:`bucket` Module
------------------------
.. automodule:: nova.objectstore.bucket
:members:
:undoc-members:
:show-inheritance:
The :mod:`handler` Module
-------------------------
.. automodule:: nova.objectstore.handler
:members:
:undoc-members:
:show-inheritance:
The :mod:`image` Module
-----------------------
.. automodule:: nova.objectstore.image
:members:
:undoc-members:
:show-inheritance:
The :mod:`stored` Module
------------------------
.. automodule:: nova.objectstore.stored
:members:
:undoc-members:
:show-inheritance:
RELATED TESTS
-------------
The :mod:`objectstore_unittest` Module
--------------------------------------
.. automodule:: nova.tests.objectstore_unittest
:members:
:undoc-members:
:show-inheritance:

29
docs/packages.rst
View File

@@ -1,29 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
nova Packages & Dependencies
============================
Nova is being built on Ubuntu Lucid.
The following packages are required:
apt-get install python-ipy, python-libvirt, python-boto, python-pycurl, python-twisted, python-daemon, python-redis, python-carrot, python-lockfile
In addition you need to install python:
* python-gflags - http://code.google.com/p/python-gflags/

31
docs/storage.rst
View File

@@ -1,31 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Storage in the Nova Cloud
=========================
There are three primary classes of storage in a nova cloud environment:
* Ephemeral Storage (local disk within an instance)
* Volume Storage (network-attached FS)
* Object Storage (redundant KVS with locality and MR)
.. toctree::
:maxdepth: 2
volume
objectstore

45
docs/volume.rst
View File

@@ -1,45 +0,0 @@
..
Copyright 2010 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Volume Documentation
====================
Nova uses ata-over-ethernet (AoE) to export storage volumes from multiple storage nodes. These AoE exports are attached (using libvirt) directly to running instances.
Nova volumes are exported over the primary system VLAN (usually VLAN 1), and not over individual VLANs.
AoE exports are numbered according to a "shelf and blade" syntax. In order to avoid collisions, we currently perform an AoE-discover of existing exports, and then grab the next unused number. (This obviously has race condition problems, and should be replaced by allocating a shelf-id to each storage node.)
The underlying volumes are LVM logical volumes, created on demand within a single large volume group.
The :mod:`storage` Module
-------------------------
.. automodule:: nova.volume.storage
:members:
:undoc-members:
:show-inheritance:
The :mod:`storage_unittest` Module
----------------------------------
.. automodule:: nova.tests.storage_unittest
:members:
:undoc-members:
:show-inheritance:

26
nova/endpoint/cloud.py
View File

@@ -452,21 +452,21 @@ class CloudController(object):
def format_addresses(self, context):
addresses = []
# TODO(vish): move authorization checking into network.py
for address in self.network.host_objs:
#logging.debug(address_record)
address_rv = {
'public_ip': address['address'],
'instance_id' : address.get('instance_id', 'free')
}
if context.user.is_admin():
address_rv['instance_id'] = "%s (%s, %s)" % (
address['instance_id'],
address['user_id'],
address['project_id'],
)
# TODO(vish): implement a by_project iterator for addresses
if (context.user.is_admin() or
address['project_id'] == self.project.id):
address_rv = {
'public_ip': address['address'],
'instance_id' : address.get('instance_id', 'free')
}
if context.user.is_admin():
address_rv['instance_id'] = "%s (%s, %s)" % (
address['instance_id'],
address['user_id'],
address['project_id'],
)
addresses.append(address_rv)
# logging.debug(addresses)
return {'addressesSet': addresses}
@rbac.allow('netadmin')

3
nova/rpc.py
View File

@@ -151,6 +151,7 @@ class TopicPublisher(Publisher):
def __init__(self, connection=None, topic="broadcast"):
self.routing_key = topic
self.exchange = FLAGS.control_exchange
self.durable = False
super(TopicPublisher, self).__init__(connection=connection)
@@ -242,7 +243,7 @@ def send_message(topic, message, wait=True):
consumer.register_callback(generic_response)
publisher = messaging.Publisher(connection=Connection.instance(),
exchange="nova",
exchange=FLAGS.control_exchange,
exchange_type="topic",
routing_key=topic)
publisher.send(message)

2
nova/tests/network_unittest.py
View File

@@ -137,7 +137,7 @@ class NetworkTestCase(test.TrialTestCase):
self.dnsmasq.release_ip(mac3, address3, hostname, net.bridge_name)
net = network.get_project_network("project0", "default")
rv = network.deallocate_ip(secondaddress)
self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name)
self.dnsmasq.release_ip(mac, secondaddress, hostname, net.bridge_name)
def test_release_before_deallocate(self):
pass

68
nova/tests/objectstore_unittest.py
View File

@@ -27,6 +27,7 @@ from nova import flags
from nova import objectstore
from nova import test
from nova.auth import users
from nova.exception import NotEmpty, NotFound, NotAuthorized
FLAGS = flags.FLAGS
@@ -96,49 +97,37 @@ class ObjectStoreTestCase(test.BaseTestCase):
# another user is not authorized
self.context.user = self.um.get_user('user2')
self.context.project = self.um.get_project('proj2')
self.assert_(bucket.is_authorized(self.context) == False)
self.assertFalse(bucket.is_authorized(self.context))
# admin is authorized to use bucket
self.context.user = self.um.get_user('admin_user')
self.context.project = None
self.assert_(bucket.is_authorized(self.context))
self.assertTrue(bucket.is_authorized(self.context))
# new buckets are empty
self.assert_(bucket.list_keys()['Contents'] == [])
self.assertTrue(bucket.list_keys()['Contents'] == [])
# storing keys works
bucket['foo'] = "bar"
self.assert_(len(bucket.list_keys()['Contents']) == 1)
self.assertEquals(len(bucket.list_keys()['Contents']), 1)
self.assert_(bucket['foo'].read() == 'bar')
self.assertEquals(bucket['foo'].read(), 'bar')
# md5 of key works
self.assert_(bucket['foo'].md5 == hashlib.md5('bar').hexdigest())
self.assertEquals(bucket['foo'].md5, hashlib.md5('bar').hexdigest())
# deleting non-empty bucket throws exception
exception = False
try:
bucket.delete()
except:
exception = True
self.assert_(exception)
# deleting non-empty bucket should throw a NotEmpty exception
self.assertRaises(NotEmpty, bucket.delete)
# deleting key
del bucket['foo']
# deleting empty button
# deleting empty bucket
bucket.delete()
# accessing deleted bucket throws exception
exception = False
try:
objectstore.bucket.Bucket('new_bucket')
except:
exception = True
self.assert_(exception)
self.assertRaises(NotFound, objectstore.bucket.Bucket, 'new_bucket')
def test_images(self):
self.context.user = self.um.get_user('user1')
@@ -167,37 +156,4 @@ class ObjectStoreTestCase(test.BaseTestCase):
# verify image permissions
self.context.user = self.um.get_user('user2')
self.context.project = self.um.get_project('proj2')
self.assert_(my_img.is_authorized(self.context) == False)
# class ApiObjectStoreTestCase(test.BaseTestCase):
# def setUp(self):
# super(ApiObjectStoreTestCase, self).setUp()
# FLAGS.fake_users = True
# FLAGS.buckets_path = os.path.join(tempdir, 'buckets')
# FLAGS.images_path = os.path.join(tempdir, 'images')
# FLAGS.ca_path = os.path.join(os.path.dirname(__file__), 'CA')
#
# self.users = users.UserManager.instance()
# self.app = handler.Application(self.users)
#
# self.host = '127.0.0.1'
#
# self.conn = boto.s3.connection.S3Connection(
# aws_access_key_id=user.access,
# aws_secret_access_key=user.secret,
# is_secure=False,
# calling_format=boto.s3.connection.OrdinaryCallingFormat(),
# port=FLAGS.s3_port,
# host=FLAGS.s3_host)
#
# self.mox.StubOutWithMock(self.ec2, 'new_http_connection')
#
# def tearDown(self):
# FLAGS.Reset()
# super(ApiObjectStoreTestCase, self).tearDown()
#
# def test_describe_instances(self):
# self.expect_http()
# self.mox.ReplayAll()
#
# self.assertEqual(self.ec2.get_all_instances(), [])
self.assertFalse(my_img.is_authorized(self.context))