[test] ConvertedExtracted openstack recipe installation; added local http server for serving RPMs

test/cookbooks/__init__.py

@@ -4,7 +4,7 @@ import unittest
 from devops.model import Environment, Network, Node, Disk, Cdrom, Interface
 from devops.controller import Controller
 from devops.driver.libvirt import Libvirt
-from devops.helpers import tcp_ping, wait, TimeoutError, ssh
+from devops.helpers import tcp_ping, wait, TimeoutError, ssh, http_server
 import traceback
 
 import simplejson as json
@@ -30,6 +30,8 @@ class Ci:
 
             try:
                 self.environment = devops.load(environment_id)
+                self.node = self.environment.nodes[0]
+                self.network = self.environment.networks[0]
                 logger.info("Successfully loaded existing environment")
             except Exception, e:
                 logger.error("Failed to load existing cookbooks environment: " + str(e) + "\n" + traceback.format_exc())
@@ -65,6 +67,8 @@ class Ci:
             return False
 
         self.environment = environment
+        self.node = node
+        self.network = network
 
         try:
             logger.info("Starting test node")
@@ -80,12 +84,14 @@ class Ci:
             repo.write("""
 [mirantis]
 name=Mirantis repository
-baseurl=http://twin0d.srt.mirantis.net/centos62
+baseurl=http://%s:8000
 enabled=1
 gpgcheck=0
-            """)
+            """ % network.ip_addresses[1])
             repo.close()
 
+            remote.execute('yum makecache')
+
             logger.info("Disabling firewall")
 
             remote.execute('service iptables save')
@@ -94,7 +100,7 @@ gpgcheck=0
 
             logger.info("Creating snapshot 'blank'")
 
-            node.save_snapshot('blank')
+            node.save_snapshot('empty')
 
             logger.info("Test node is ready at %s" % node.ip_address)
 
@@ -133,14 +139,53 @@ gpgcheck=0
 
         return True
 
+    def setup_repository(self):
+        remote = ssh(self.node.ip_address, username='root', password='r00tme')
+        repo = remote.open('/etc/yum.repos.d/mirantis.repo', 'w')
+        repo.write("""
+[mirantis]
+name=Mirantis repository
+baseurl=http://%s:%d
+enabled=1
+gpgcheck=0
+        """ % (self.network.ip_addresses[1], self.repository_server.port))
+        repo.close()
+
+        remote.execute('yum makecache')
+
+
+    def start_environment(self):
+        self.repository_server = http_server(
+            os.path.abspath(
+                os.path.join(
+                    os.path.dirname(__file__), "..", "..",
+                    "build", "packages", "centos", "Packages"
+                ) 
+            )
+        )
+
+        self.setup_repository()
+
+        self.node.save_snapshot('blank', force=True)
+
+        return True
+
+    def shutdown_environment(self):
+        if hasattr(self, 'repository_server'):
+            self.repository_server.stop()
+
+        return True
 
 ci = None
 
 def setUp():
     if not ci.setup_environment():
         raise Exception, "Failed to setup cookbooks environment"
+    if not ci.start_environment():
+        raise Exception, "Failed to run cookboks environment"
 
 def tearDown():
+    ci.shutdown_environment()
     if not ci.environment_cache_file:
         ci.destroy_environment()
 
@@ -164,23 +209,27 @@ class CookbookTestCase(unittest.TestCase):
 
         klass.remote = ssh(klass.ip, username='root', password='r00tme')
 
-        snapshot = klass.__name__
+        try:
+            snapshot = klass.__name__
 
-        if not os.environ.get('DEVELOPMENT') or not snapshot in klass.node.snapshots:
+            if not os.environ.get('DEVELOPMENT') or not snapshot in klass.node.snapshots:
-            logger.info('Setting up state for %s' % klass.__name__)
+                logger.info('Setting up state for %s' % klass.__name__)
 
-            if snapshot in klass.node.snapshots:
+                if snapshot in klass.node.snapshots:
-                klass.node.delete_snapshot(snapshot)
+                    klass.node.delete_snapshot(snapshot)
 
-            klass.node.restore_snapshot('blank')
+                klass.node.restore_snapshot('blank')
-            klass.remote.reconnect()
+                klass.remote.reconnect()
 
-            klass.upload_cookbooks(klass.cookbooks)
+                klass.upload_cookbooks(klass.cookbooks)
-            klass.setUpState()
+                klass.setUpState()
 
-            klass.node.save_snapshot(snapshot)
+                klass.node.save_snapshot(snapshot)
 
-            logger.info('Finished state for %s' % klass.__name__)
+                logger.info('Finished state for %s' % klass.__name__)
+        except:
+            # klass.http_server.stop()
+            raise
 
     @classmethod
     def upload_cookbooks(klass, cookbooks):
@@ -198,6 +247,8 @@ cookbook_path "/opt/os-cookbooks"
 
     @classmethod
     def chef_solo(klass, attributes={}):
+        ci.setup_repository()
+
         recipes = attributes.get('recipes') or attributes.get('run_list')
         logger.info('Running Chef with recipes: %s' % (', '.join(recipes)))
 

test/cookbooks/openstack_common.py

@@ -0,0 +1,122 @@
+
+class OpenstackCommon(object):
+    mysql_port = 3306
+
+    @classmethod
+    def setUpMysql(klass, reuse_cached=True):
+        if not klass.node.has_snapshot('openstack-mysql') or not reuse_cached:
+            klass.upload_cookbooks(['mysql'])
+
+            klass.chef_solo({
+                'recipes': ['mysql::server'],
+                'mysql': {
+                    'port': klass.mysql_port,
+                    'db_maker_password': 'secret'
+                },
+            })
+
+            if not klass.node.has_snapshot('openstack-mysql'):
+                klass.node.save_snapshot('openstack-mysql')
+        else:
+            klass.node.restore_snapshot('openstack-mysql')
+            klass.remote.reconnect()
+
+
+    keystone_admin_port  = 37376
+    keystone_public_port = 5000
+
+    @classmethod
+    def setUpKeystone(klass, reuse_cached=True):
+        if not klass.node.has_snapshot('openstack-keystone') or not reuse_cached:
+            klass.setUpMysql()
+
+            klass.upload_cookbooks(['chef-resource-groups', 'database', 'keystone'])
+
+            klass.chef_solo({
+                'recipes': ['keystone::server'],
+                'mysql': {
+                    'admin': {
+                        'host': str(klass.ip),
+                        'port': klass.mysql_port,
+                        'username': 'db_maker',
+                        'password': 'test'
+                    }
+                 },
+                'keystone': {
+                    'db': {
+                        'password': 'secret'
+                    },
+                    'admin_token': 'secret',
+                    'service_tenant': 'service',
+                    'admin_tenant': 'admin',
+                    'admin_user': 'admin',
+                    'admin_password': 'admin',
+                    'admin_role': 'admin',
+                    'admin_url': 'http://%s:%s/' % (klass.ip,
+                            klass.keystone_public_port),
+                    'public_url': 'http://%s:%s/' % (klass.ip,
+                            klass.keystone_public_port),
+                    'internal_url': 'http://%s:%s/' % (klass.ip,
+                            klass.keystone_public_port),
+                    'admin': {
+                        'host': klass.ip,
+                        'admin_port': klass.keystone_admin_port,
+                        'service_port': klass.keystone_public_port,
+                    },
+                    'public': {
+                        'host': klass.ip,
+                        'admin_port': klass.keystone_admin_port,
+                        'service_port': klass.keystone_public_port,
+                    },
+                },
+            })
+
+            if not klass.node.has_snapshot('openstack-keystone'):
+                klass.node.save_snapshot('openstack-keystone')
+        else:
+            klass.node.restore_snapshot('openstack-keystone')
+            klass.remote.reconnect()
+
+
+    glance_registry_port = 9191
+
+    @classmethod
+    def setUpGlanceRegistry(klass, reuse_cached=True):
+        if not klass.node.has_snapshot('openstack-glance-registry') or not reuse_cached:
+            klass.setUpKeystone()
+
+            klass.upload_cookbooks(['chef-resource-groups', 'database', 'keystone', 'glance'])
+
+            klass.chef_solo({
+                'recipes': ['glance::registry'],
+                'glance': {
+                    'registry': {
+                        'port': klass.glance_registry_port,
+                    },
+                },
+                'services': {
+                    'glance_registry': {
+                        'endpoints': {
+                            'mysql': {
+                                'host': str(klass.ip),
+                                'port': klass.mysql_port,
+                                'username': 'db_maker',
+                                'password': 'secret',
+                            },
+                            'keystone_admin': {
+                                'host': str(klass.ip),
+                                'port': klass.keystone_admin_port,
+                                'username': 'admin',
+                                'token': 'keystone_secret',
+                            },
+                        }
+                    },
+                }
+            })
+            
+            if not klass.node.has_snapshot('openstack-glance-registry'):
+                klass.node.save_snapshot('openstack-glance-registry')
+        else:
+            klass.node.restore_snapshot('openstack-glance-registry')
+            klass.remote.reconnect()
+

test/cookbooks/test_keystone.py

@@ -1,82 +1,37 @@
 from . import CookbookTestCase
+from openstack_common import OpenstackCommon
+
 from devops.helpers import tcp_ping
 from integration.helpers import HTTPClient
 import os
 import simplejson as json
 
-class TestKeystone(CookbookTestCase):
+def find(predicate, sequence):
-    cookbooks = ['chef-resource-groups', 'mysql', 'database', 'keystone']
+    "Returns first element of sequence for which predicate is true or None"
+    for item in sequence:
+        if predicate(item):
+            return item
 
-    mysql_port = 3306
+    return None
-
-    keystone_admin_port  = 37376
-    keystone_public_port = 5000
 
+class TestKeystone(CookbookTestCase, OpenstackCommon):
+    cookbooks = ['chef-resource-groups', 'database', 'keystone']
 
     @classmethod
     def setUpState(klass):
-        klass.chef_solo({
+        klass.setUpMysql()
-            'recipes': ['mysql::server'],
+        klass.setUpKeystone(reuse_cached=False)
-            'mysql': {
-                'port': klass.mysql_port,
-                'db_maker_password': 'secret'
-            },
-        })
-
-
-        klass.chef_solo({
-            'recipes': ['keystone::server'],
-            'mysql': {
-                'admin': {
-                    'host': str(klass.ip),
-                    'port': klass.mysql_port,
-                    'username': 'db_maker',
-                    'password': 'test'
-                }
-             },
-            'keystone': {
-                'db': {
-                    'password': 'secret'
-                },
-                'admin_token': 'secret',
-                'service_tenant': 'service',
-                'admin_tenant': 'admin',
-                'admin_user': 'admin',
-                'admin_password': 'admin',
-                'admin_role': 'admin',
-                'admin_url': 'http://%s:%s/' % (klass.ip,
-                        klass.keystone_public_port),
-                'public_url': 'http://%s:%s/' % (klass.ip,
-                        klass.keystone_public_port),
-                'internal_url': 'http://%s:%s/' % (klass.ip,
-                        klass.keystone_public_port),
-                'admin': {
-                    'host': klass.ip,
-                    'admin_port': klass.keystone_admin_port,
-                    'service_port': klass.keystone_public_port,
-                },
-                'public': {
-                    'host': klass.ip,
-                    'admin_port': klass.keystone_admin_port,
-                    'service_port': klass.keystone_public_port,
-                },
-            },
-        })
 
     def test_public_port_open(self):
         assert tcp_ping(self.ip, self.keystone_public_port)
 
     def test_keystone_identity_service_registered(self):
-        http = HTTPClient()
-
         keystone_url = "http://%s:%d" % (self.ip, self.keystone_admin_port)
 
+        http = HTTPClient()
         services = json.loads(http.get(keystone_url + "/v2.0/OS-KSADM/services"))['OS-KSADM:services']
-        keystone_service = None
+
-        for service in services:
+        keystone_service = find(lambda service: service['name'] == 'keystone', services)
-            if service['name'] == 'keystone':
-                keystone_service = service
-                break
 
         assert keystone_service, "Keystone service is not registered"
         assert keystone_service['type'] == 'identity', \

test/cookbooks/test_mysql.py

@@ -1,19 +1,13 @@
 from . import CookbookTestCase
+from openstack_common import OpenstackCommon
 from devops.helpers import tcp_ping
 
-class TestMysql(CookbookTestCase):
+class TestMysql(CookbookTestCase, OpenstackCommon):
     cookbooks = ['mysql']
 
-    mysql_port = 3306
-
     @classmethod
     def setUpState(klass):
-        klass.chef_solo({
+        klass.setUpMysql(reuse_cached=False)
-            'recipes': ['mysql::server'],
-            'mysql': {
-                'port': klass.mysql_port
-            }
-        })
 
     def test_mysql_deploy(self):
         assert tcp_ping(self.ip, self.mysql_port)

test/module.mk

@@ -35,7 +35,7 @@ clean-integration-test:
 
 
 .PHONY: test-cookbooks
-test-cookbooks: $/environment-id-cookbooks
+test-cookbooks: $/environment-id-cookbooks $(centos.packages)/Packages/repodata/repomd.xml
 	python test/integration_test.py -l $(LEVEL) --cache-file $(abspath $<) --iso $(image.centos.url) --suite cookbooks test $(NOSEARGS)
 
 $/environment-id-cookbooks: