Cinder component was added to anvil

Change-Id: Ieace320bc3bed4669fb677cde5d6dcc96d13e516

.gitignore

@@ -17,3 +17,6 @@ TAGS
 *_flymake.*
 *~
 .anvil_bootstrapped
+.metadata/
+.project
+.pydevproject

anvil/components/cinder.py

@@ -0,0 +1,184 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import io
+
+from anvil import cfg
+from anvil import colorizer
+from anvil import components as comp
+from anvil import log as logging
+from anvil import shell as sh
+from anvil import utils
+
+from anvil.components.helpers import db as dbhelper
+from anvil.components.helpers import keystone as khelper
+from anvil.components.helpers import cinder as chelper
+
+LOG = logging.getLogger(__name__)
+
+# Copies from helpers
+API_CONF = chelper.API_CONF
+
+# Paste configuration
+PASTE_CONF = chelper.PASTE_CONF
+
+CONFIGS = [PASTE_CONF, API_CONF]
+
+# This db will be dropped and created
+DB_NAME = "cinder"
+
+# Sync db command
+SYNC_DB_CMD = [sh.joinpths('$BIN_DIR', 'cinder-manage'),
+                # Available commands:
+                'db', 'sync']
+
+BIN_DIR = 'bin'
+
+class CinderUninstaller(comp.PythonUninstallComponent):
+    def __init__(self, *args, **kargs):
+        comp.PythonUninstallComponent.__init__(self, *args, **kargs)
+        self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
+
+
+class CinderInstaller(comp.PythonInstallComponent):
+    def __init__(self, *args, **kargs):
+        comp.PythonInstallComponent.__init__(self, *args, **kargs)
+        self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
+        self.conf_maker = chelper.ConfConfigurator(self)
+
+    @property
+    def config_files(self):
+        return list(CONFIGS)
+
+    def post_install(self):
+        comp.PythonInstallComponent.post_install(self)
+        if self.get_bool_option('db-sync'):
+            self._setup_db()
+            self._sync_db()
+
+    def _filter_pip_requires(self, fn, lines):
+        return [l for l in lines
+            # Take out entries that aren't really always needed or are
+            # resolved/installed by anvil during installation in the first
+            # place..
+            if not utils.has_any(l.lower(), 'oslo.config')]
+
+    def _setup_db(self):
+        dbhelper.drop_db(distro=self.distro,
+                         dbtype=self.get_option('db', 'type'),
+                         dbname=DB_NAME,
+                         **utils.merge_dicts(self.get_option('db'),
+                                             dbhelper.get_shared_passwords(self)))
+        dbhelper.create_db(distro=self.distro,
+                           dbtype=self.get_option('db', 'type'),
+                           dbname=DB_NAME,
+                           **utils.merge_dicts(self.get_option('db'),
+                                               dbhelper.get_shared_passwords(self)))
+
+    def _sync_db(self):
+        LOG.info("Syncing cinder to database: %s", colorizer.quote(DB_NAME))
+        cmds = [{'cmd': SYNC_DB_CMD, 'run_as_root': True}]
+        utils.execute_template(*cmds, cwd=self.bin_dir, params=self.config_params(None))
+
+    def source_config(self, config_fn):
+        if config_fn == API_CONF:
+            config_fn = 'cinder.conf.sample'
+        fn = sh.joinpths(self.get_option('app_dir'), 'etc', "cinder", config_fn)
+        return (fn, sh.load_file(fn))
+
+    def _fetch_keystone_params(self):
+        params = khelper.get_shared_params(ip=self.get_option('ip'),
+                                           service_user='cinder',
+                                           **utils.merge_dicts(self.get_option('keystone'),
+                                                               khelper.get_shared_passwords(self)))
+        return {
+            'auth_host': params['endpoints']['admin']['host'],
+            'auth_port': params['endpoints']['admin']['port'],
+            'auth_protocol': params['endpoints']['admin']['protocol'],
+            # This uses the public uri not the admin one...
+            'auth_uri': params['endpoints']['public']['uri'],
+            'admin_tenant_name': params['service_tenant'],
+            'admin_user': params['service_user'],
+            'admin_password': params['service_password'],
+
+            'service_host': params['endpoints']['internal']['host'],
+            'service_port': params['endpoints']['internal']['port'],
+            'service_protocol': params['endpoints']['internal']['protocol'],
+            'auth_version': 'v2.0'
+
+        }
+
+    def _config_adjust(self, contents, name):
+        if name == PASTE_CONF:
+            return self._config_adjust_paste(contents, name)
+        elif name == API_CONF:
+            return self._generate_cinder_conf(name)
+        else:
+            return contents
+
+    def _config_adjust_paste(self, contents, fn):
+        with io.BytesIO(contents) as stream:
+            config = cfg.create_parser(cfg.RewritableConfigParser, self)
+            config.readfp(stream)
+            for (k, v) in self._fetch_keystone_params().items():
+                config.set('filter:authtoken', k, v)
+            contents = config.stringify(fn)
+        return contents
+
+    def _config_param_replace(self, config_fn, contents, parameters):
+        if config_fn in [PASTE_CONF, API_CONF]:
+            # We handle these ourselves
+            return contents
+        else:
+            return comp.PythonInstallComponent._config_param_replace(self, config_fn, contents, parameters)
+
+    def _generate_cinder_conf(self, fn):
+        LOG.debug("Generating dynamic content for cinder: %s.", (fn))
+        return self.conf_maker.generate(fn)
+
+    def config_params(self, config_fn):
+        # These be used to fill in the configuration params
+        mp = comp.PythonInstallComponent.config_params(self, config_fn)
+        mp['CFG_FILE'] = sh.joinpths(self.get_option('cfg_dir'), API_CONF)
+        mp['BIN_DIR'] = self.bin_dir
+        return mp
+
+
+class CinderRuntime(comp.PythonRuntime):
+    def __init__(self, *args, **kargs):
+        comp.PythonRuntime.__init__(self, *args, **kargs)
+        self.bin_dir = sh.joinpths(self.get_option('app_dir'), BIN_DIR)
+        self.config_path = sh.joinpths(self.get_option('cfg_dir'), API_CONF)
+
+    @property
+    def applications(self):
+        apps = []
+        for (name, _values) in self.subsystems.items():
+            name = "cinder-%s" % (name.lower())
+            path = sh.joinpths(self.bin_dir, name)
+            if sh.is_executable(path):
+                apps.append(comp.Program(name, path, argv=self._fetch_argv(name)))
+        return apps
+
+    def app_params(self, program):
+        params = comp.PythonRuntime.app_params(self, program)
+        params['CFG_FILE'] = self.config_path
+        return params
+
+    def _fetch_argv(self, name):
+        return [
+            '--config-file', '$CFG_FILE',
+        ]

anvil/components/helpers/cinder.py

@@ -0,0 +1,107 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import weakref
+
+from anvil import cfg
+from anvil import utils
+from anvil import log as logging
+
+from anvil.components.helpers import nova as nhelper
+from anvil.components.helpers import rabbit as rbhelper
+from anvil.components.helpers import db as dbhelper
+
+LOG = logging.getLogger(__name__)
+
+# Special generated conf
+API_CONF = 'cinder.conf'
+
+# Paste configuration
+PASTE_CONF = 'api-paste.ini'
+
+# Message queue types to there internal 'canonicalized' name
+MQ_TYPES = {
+    'qpid': 'qpid',
+    'qpidd': 'qpid',
+    'rabbit': 'rabbit',
+    'rabbit-mq': 'rabbit',
+}
+
+# This db will be dropped then created
+DB_NAME = 'cinder'
+
+def get_shared_params(ip, api_host, api_port=8776, protocol='http', **kwargs):
+    mp = {}
+    mp['service_host'] = ip
+
+    # Uri's of the various cinder endpoints
+    mp['endpoints'] = {
+        'volume': {
+            'uri': utils.make_url(protocol, api_host, api_port, "v2"),
+            'port': api_port,
+            'host': api_host,
+            'protocol': protocol,
+        },
+        'internal': {
+        }
+    }
+
+    return mp
+
+
+class ConfConfigurator(object):
+
+    def __init__(self, installer):
+        self.installer = weakref.proxy(installer)
+
+    def generate(self, fn):
+
+        backing = cfg.create_parser(cfg.BuiltinConfigParser, self.installer)
+
+        # Everything built goes in here
+        cinder_conf = cfg.DefaultConf(backing)
+
+        # Used more than once so we calculate it ahead of time
+        hostip = self.installer.get_option('ip')
+
+        # How is your message queue setup?
+        mq_type = nhelper.canon_mq_type(self.installer.get_option('mq-type'))
+        if mq_type == 'rabbit':
+            cinder_conf.add('rabbit_host', self.installer.get_option('rabbit', 'host', default_value=hostip))
+            cinder_conf.add('rabbit_password', rbhelper.get_shared_passwords(self.installer)['pw'])
+            cinder_conf.add('rabbit_userid', self.installer.get_option('rabbit', 'user_id'))
+
+        # Setup your sql connection
+        dbdsn = dbhelper.fetch_dbdsn(
+            dbname=DB_NAME,
+            utf8=True,
+            dbtype=self.installer.get_option('db', 'type'),
+            **utils.merge_dicts(self.installer.get_option('db'),
+            dbhelper.get_shared_passwords(self.installer)))
+        cinder_conf.add('sql_connection', dbdsn)
+
+        # Auth will be using keystone
+        cinder_conf.add('auth_strategy', 'keystone')
+
+        # Where our paste config is
+        cinder_conf.add('api_paste_config', self.installer.target_config(PASTE_CONF))
+
+        # Extract to finish
+        return backing.stringify(fn)
+
+    def _get_content(self, cinder_conf):
+        generated_content = cinder_conf.generate()
+        return generated_content

anvil/components/helpers/nova.py

@@ -74,7 +74,6 @@ def canon_virt_driver(virt_driver):
 def get_shared_params(ip, protocol,
                       api_host, api_port,
                       s3_host, s3_port,
-                      volume_host, volume_port,
                       ec2_host, ec2_port,
                       ec2_admin_host, ec2_admin_port, **kwargs):
     mp = {}
@@ -94,12 +93,6 @@ def get_shared_params(ip, protocol,
             'host': ec2_host,
             'protocol': protocol,
         },
-        'volume': {
-            'uri': utils.make_url(protocol, volume_host, volume_port, "v1"),
-            'port': volume_port,
-            'host': volume_host,
-            'protocol': protocol,
-        },
         's3': {
             'uri': utils.make_url(protocol, s3_host, s3_port),
             'port': s3_port,
@@ -306,10 +299,6 @@ class ConfConfigurator(object):
         # Setup nova network/settings
         self._configure_network_settings(nova_conf)
 
-        # Setup nova volume/settings
-        if self.installer.get_option('volumes'):
-            self._configure_vols(nova_conf)
-
         # The ip of where we are running
         nova_conf.add('my_ip', hostip)
 
@@ -465,15 +454,6 @@ class ConfConfigurator(object):
         nova_conf.add('vncserver_listen', self.installer.get_option('vncserver_listen', default_value='127.0.0.1'))
         nova_conf.add('vncserver_proxyclient_address', self.installer.get_option('vncserver_proxyclient_address', default_value='127.0.0.1'))
 
-    # Fixes up your nova volumes
-    def _configure_vols(self, nova_conf):
-        nova_conf.add('volume_group', self.installer.get_option('volume_group'))
-        vol_name_tpl = self.installer.get_option('volume_name_prefix') + self.installer.get_option('volume_name_postfix')
-        if not vol_name_tpl:
-            vol_name_tpl = 'volume-%08x'
-        nova_conf.add('volume_name_template', vol_name_tpl)
-        nova_conf.add('iscsi_helper', 'tgtadm')
-
     def _configure_quantum(self, nova_conf):
         params = khelper.get_shared_params(
             ip=self.installer.get_option('ip'),

anvil/components/keystone.py

@@ -30,6 +30,7 @@ from anvil.components.helpers import glance as ghelper
 from anvil.components.helpers import keystone as khelper
 from anvil.components.helpers import nova as nhelper
 from anvil.components.helpers import quantum as qhelper
+from anvil.components.helpers import cinder as chelper
 
 LOG = logging.getLogger(__name__)
 
@@ -237,6 +238,7 @@ class KeystoneRuntime(comp.PythonRuntime):
             params['glance'] = ghelper.get_shared_params(ip=self.get_option('ip'), **self.get_option('glance'))
             params['nova'] = nhelper.get_shared_params(ip=self.get_option('ip'), **self.get_option('nova'))
             params['quantum'] = qhelper.get_shared_params(ip=self.get_option('ip'), **self.get_option('quantum'))
+            params['cinder'] = chelper.get_shared_params(ip=self.get_option('ip'), **self.get_option('cinder'))
             wait_urls = [
                 params['keystone']['endpoints']['admin']['uri'],
                 params['keystone']['endpoints']['public']['uri'],

conf/components/cinder.yaml

@@ -0,0 +1,41 @@
+# Settings for component cinder-client
+---
+
+# Where we download this from...
+get_from: "git://github.com/openstack/cinder.git?tag=2013.1"
+
+# Host and ports for the different cinder services
+api_host: "$(auto:ip)"
+api_port: 8776
+protocol: http
+
+# Needed for setting up your database
+db:
+  type: "$(db:type)"
+  user: "$(db:user)"
+  host: "$(db:host)"
+  port: "$(db:port)"
+
+# Interactions with keystone are via the following settings
+keystone:
+  auth_host: "$(keystone:auth_host)"
+  auth_port: "$(keystone:auth_port)"
+  auth_proto: "$(keystone:auth_proto)"
+  service_host: "$(keystone:service_host)"
+  service_port: "$(keystone:service_port)"
+  service_proto: "$(keystone:service_proto)"
+
+# Rabbit mq hookins
+rabbit:
+  user_id: "$(rabbit-mq:user_id)"
+  host: "$(rabbit-mq:host)"
+
+# This is needed to allow installs based on personas
+wanted_passwords:
+  rabbit: 'rabbit user'
+  service_token: 'service admin token'
+  admin_password: 'keystone admin user'
+  service_password: 'service authentication password'
+  sql: "database user"
+
+...

conf/components/keystone.yaml

@@ -49,8 +49,6 @@ nova:
   api_port: "$(nova:api_port)"
   s3_host: "$(nova:s3_host)"
   s3_port: "$(nova:s3_port)"
-  volume_host: "$(nova:volume_host)"
-  volume_port: "$(nova:volume_port)"
   ec2_host: "$(nova:ec2_host)"
   ec2_port: "$(nova:ec2_port)"
   ec2_admin_host: "$(nova:ec2_admin_host)"
@@ -61,6 +59,10 @@ quantum:
   api_host: "$(quantum:api_host)"
   api_port: "$(quantum:api_port)"
 
+cinder:
+  api_host: "$(cinder:api_host)"
+  api_port: "$(cinder:api_port)"
+
 # This is needed to allow installs based on personas
 wanted_passwords:
   rabbit: 'rabbit user'

conf/components/nova.yaml

@@ -120,13 +120,6 @@ vncserver_listen: 127.0.0.1
 vncserver_proxyclient_address: ""
 xvpvncproxy_url: "http://$(auto:ip):6081/console"
 
-# Not currently working (to be replaced by cinder)
-volume_backing_file: ""
-volume_backing_file_size: 2052M
-volume_group: nova-volumes
-volume_name_postfix: "%08x"
-volume_name_prefix: "volume-"
-
 # Package time patches
 patches:
   package:

conf/distros/rhel.yaml

@@ -52,6 +52,16 @@ commands:
         status: service rabbitmq-server status
         stop: service rabbitmq-server stop
 components:
+    cinder:
+        action_classes:
+            install: anvil.components.cinder:CinderInstaller
+            package: anvil.packaging.rpm:PythonPackager
+            running: anvil.components.cinder:CinderRuntime
+            test: anvil.components:PythonTestingComponent
+            coverage: anvil.components:PythonTestingComponent
+            uninstall: anvil.components.cinder:CinderUninstaller
+        pips:
+            - name: hp3parclient
     cinder-client:
         action_classes:
             install: anvil.components:PythonInstallComponent

conf/personas/in-a-box/basic-web.yaml

@@ -15,6 +15,7 @@ components:
 - quantum-client
 - swift-client # Seems only needed for horizon?
 - quantum
+- cinder
 - no-vnc
 - nova
 - nova-client
@@ -41,6 +42,8 @@ options:
         enable-pki: false
     horizon:
         make-blackhole: true
+    cinder:
+        db-sync: true
     quantum:
         db-sync: true
 subsystems:
@@ -64,6 +67,8 @@ subsystems:
     - xvpvncproxy
     quantum:
     - server
+    cinder:
+    - all
 supports:
 - rhel
 ...

conf/templates/keystone/init_what.yaml

@@ -25,9 +25,9 @@ endpoints:
     public_url: "$glance.endpoints.public.uri"
     region: RegionOne
 -   service: volume
-    admin_url: "${nova.endpoints.volume.uri}/%(tenant_id)s"
-    internal_url: "${nova.endpoints.volume.uri}/%(tenant_id)s"
-    public_url: "${nova.endpoints.volume.uri}/%(tenant_id)s"
+    admin_url: "${cinder.endpoints.volume.uri}/%(tenant_id)s"
+    internal_url: "${cinder.endpoints.volume.uri}/%(tenant_id)s"
+    public_url: "${cinder.endpoints.volume.uri}/%(tenant_id)s"
     region: RegionOne
 -   service: s3
     admin_url: "$nova.endpoints.s3.uri"
@@ -74,7 +74,7 @@ services:
 -   description: S3 Service
     name: s3
     type: s3
--   description: Nova Volume Service
+-   description: Cinder Service
     name: volume
     type: volume
 -   description: Quantum Service
@@ -118,6 +118,14 @@ users:
     - ResellerAdmin:service
     tenants:
     - service
+-   email: cinder@example.com
+    name: cinder
+    password: '$keystone.service_password'
+    roles:
+    - admin:service
+    - ResellerAdmin:service
+    tenants:
+    - service
 -   email: quantum@example.com
     name: quantum
     password: '$keystone.service_password'