diff --git a/nodepool/cmd/config_validator.py b/nodepool/cmd/config_validator.py index ed4182304..dfa96cfee 100644 --- a/nodepool/cmd/config_validator.py +++ b/nodepool/cmd/config_validator.py @@ -50,6 +50,7 @@ class ConfigValidator: 'service-name': str, 'availability-zones': [str], 'keypair': str, + 'cloud': str, 'username': str, 'password': str, 'auth-url': str, diff --git a/nodepool/fakeprovider.py b/nodepool/fakeprovider.py index bd83cc1df..32b3e6710 100644 --- a/nodepool/fakeprovider.py +++ b/nodepool/fakeprovider.py @@ -53,6 +53,26 @@ def get_fake_images_list(): return fake_images_list +BAD_CLIENT = None + + +def get_bad_client(): + global BAD_CLIENT + if BAD_CLIENT is None: + BAD_CLIENT = BadOpenstackCloud() + return BAD_CLIENT + + +FAKE_CLIENT = None + + +def get_fake_client(**kwargs): + global FAKE_CLIENT + if FAKE_CLIENT is None: + FAKE_CLIENT = FakeOpenStackCloud() + return FAKE_CLIENT + + class FakeList(object): def __init__(self, l): self._list = l @@ -143,9 +163,13 @@ class BadClient(FakeClient): self.client = BadHTTPClient() +class BadOpenstackCloud(object): + nova_client = BadClient() + + class FakeGlanceClient(object): - def __init__(self, *args, **kwargs): - self.id = 'fake-glance-id' + def __init__(self, **kwargs): + self.kwargs = kwargs self.images = get_fake_images_list() @@ -160,6 +184,11 @@ class FakeKeystoneClient(object): self.auth_token = 'fake-auth-token' +class FakeOpenStackCloud(object): + nova_client = FakeClient() + glance_client = FakeGlanceClient() + + class FakeFile(StringIO.StringIO): def __init__(self, path): StringIO.StringIO.__init__(self) @@ -236,7 +265,3 @@ class FakeJenkins(object): {u'name': u'test-view', u'url': u'https://jenkins.example.com/view/test-view/'}]} return d - - -FAKE_CLIENT = FakeClient() -BAD_CLIENT = BadClient() diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py index 2ea428a98..b4bdacf27 100644 --- a/nodepool/nodepool.py +++ b/nodepool/nodepool.py @@ -1249,10 +1249,11 @@ class NodePool(threading.Thread): p = Provider() p.name = provider['name'] newconfig.providers[p.name] = p - p.username = provider['username'] - p.password = provider['password'] - p.project_id = provider['project-id'] - p.auth_url = provider['auth-url'] + p.username = provider.get('username') + p.password = provider.get('password') + p.project_id = provider.get('project-id') + p.auth_url = provider.get('auth-url') + p.cloud = provider.get('cloud') p.service_type = provider.get('service-type') p.service_name = provider.get('service-name') p.region_name = provider.get('region-name') @@ -1397,6 +1398,7 @@ class NodePool(threading.Thread): new_pm.password != old_pm.provider.password or new_pm.project_id != old_pm.provider.project_id or new_pm.auth_url != old_pm.provider.auth_url or + new_pm.cloud != old_pm.provider.cloud or new_pm.service_type != old_pm.provider.service_type or new_pm.service_name != old_pm.provider.service_name or new_pm.max_servers != old_pm.provider.max_servers or diff --git a/nodepool/provider_manager.py b/nodepool/provider_manager.py index 1c9da64c0..0974852e7 100644 --- a/nodepool/provider_manager.py +++ b/nodepool/provider_manager.py @@ -18,18 +18,15 @@ import logging import paramiko -import novaclient -import novaclient.client -import novaclient.extension -import novaclient.v2.contrib.tenant_networks + import threading -import glanceclient -import glanceclient.client -import keystoneclient.v2_0.client as ksclient import time import requests.exceptions import sys +import shade +import novaclient + from nodeutils import iterate_timeout from task_manager import Task, TaskManager, ManagerStoppedException @@ -113,14 +110,14 @@ class NotFound(Exception): class CreateServerTask(Task): def main(self, client): - server = client.servers.create(**self.args) + server = client.nova_client.servers.create(**self.args) return str(server.id) class GetServerTask(Task): def main(self, client): try: - server = client.servers.get(self.args['server_id']) + server = client.nova_client.servers.get(self.args['server_id']) except novaclient.exceptions.NotFound: raise NotFound() return make_server_dict(server) @@ -128,52 +125,52 @@ class GetServerTask(Task): class DeleteServerTask(Task): def main(self, client): - client.servers.delete(self.args['server_id']) + client.nova_client.servers.delete(self.args['server_id']) class ListServersTask(Task): def main(self, client): - servers = client.servers.list() + servers = client.nova_client.servers.list() return [make_server_dict(server) for server in servers] class AddKeypairTask(Task): def main(self, client): - client.keypairs.create(**self.args) + client.nova_client.keypairs.create(**self.args) class ListKeypairsTask(Task): def main(self, client): - keys = client.keypairs.list() + keys = client.nova_client.keypairs.list() return [dict(id=str(key.id), name=key.name) for key in keys] class DeleteKeypairTask(Task): def main(self, client): - client.keypairs.delete(self.args['name']) + client.nova_client.keypairs.delete(self.args['name']) class CreateFloatingIPTask(Task): def main(self, client): - ip = client.floating_ips.create(**self.args) + ip = client.nova_client.floating_ips.create(**self.args) return dict(id=str(ip.id), ip=ip.ip) class AddFloatingIPTask(Task): def main(self, client): - client.servers.add_floating_ip(**self.args) + client.nova_client.servers.add_floating_ip(**self.args) class GetFloatingIPTask(Task): def main(self, client): - ip = client.floating_ips.get(self.args['ip_id']) + ip = client.nova_client.floating_ips.get(self.args['ip_id']) return dict(id=str(ip.id), ip=ip.ip, instance_id=str(ip.instance_id)) class ListFloatingIPsTask(Task): def main(self, client): - ips = client.floating_ips.list() + ips = client.nova_client.floating_ips.list() return [dict(id=str(ip.id), ip=ip.ip, instance_id=str(ip.instance_id)) for ip in ips] @@ -181,24 +178,24 @@ class ListFloatingIPsTask(Task): class RemoveFloatingIPTask(Task): def main(self, client): - client.servers.remove_floating_ip(**self.args) + client.nova_client.servers.remove_floating_ip(**self.args) class DeleteFloatingIPTask(Task): def main(self, client): - client.floating_ips.delete(self.args['ip_id']) + client.nova_client.floating_ips.delete(self.args['ip_id']) class CreateImageTask(Task): def main(self, client): # This returns an id - return str(client.servers.create_image(**self.args)) + return str(client.nova_client.servers.create_image(**self.args)) class GetImageTask(Task): def main(self, client): try: - image = client.images.get(**self.args) + image = client.nova_client.images.get(**self.args) except novaclient.exceptions.NotFound: raise NotFound() # HP returns 404, rackspace can return a 'DELETED' image. @@ -210,7 +207,7 @@ class GetImageTask(Task): class ListExtensionsTask(Task): def main(self, client): try: - resp, body = client.client.get('/extensions') + resp, body = client.nova_client.client.get('/extensions') return [x['alias'] for x in body['extensions']] except novaclient.exceptions.NotFound: # No extensions present. @@ -219,32 +216,33 @@ class ListExtensionsTask(Task): class ListFlavorsTask(Task): def main(self, client): - flavors = client.flavors.list() + flavors = client.nova_client.flavors.list() return [dict(id=str(flavor.id), ram=flavor.ram, name=flavor.name) for flavor in flavors] class ListImagesTask(Task): def main(self, client): - images = client.images.list() + images = client.nova_client.images.list() return [make_image_dict(image) for image in images] class FindImageTask(Task): def main(self, client): - image = client.images.find(**self.args) + image = client.nova_client.images.find(**self.args) return dict(id=str(image.id)) class DeleteImageTask(Task): def main(self, client): - client.images.delete(**self.args) + client.nova_client.images.delete(**self.args) class FindNetworkTask(Task): def main(self, client): - network = client.tenant_networks.find(**self.args) - return dict(id=str(network.id)) + for network in client.neutron_client.list_networks()['networks']: + if self.args['label'] == network['name']: + return dict(id=str(network['id'])) class ProviderManager(TaskManager): @@ -285,20 +283,31 @@ class ProviderManager(TaskManager): self._cloud_metadata_read = True def _getClient(self): - tenant_networks = novaclient.extension.Extension( - 'tenant_networks', novaclient.v2.contrib.tenant_networks) - args = ['1.1', self.provider.username, self.provider.password, - self.provider.project_id, self.provider.auth_url] - kwargs = {'extensions': [tenant_networks]} - if self.provider.service_type: - kwargs['service_type'] = self.provider.service_type - if self.provider.service_name: - kwargs['service_name'] = self.provider.service_name + kwargs = {} if self.provider.region_name: kwargs['region_name'] = self.provider.region_name if self.provider.api_timeout: - kwargs['timeout'] = self.provider.api_timeout - return novaclient.client.Client(*args, **kwargs) + kwargs['api_timeout'] = self.provider.api_timeout + # These are named from back when we only talked to Nova. They're + # actually compute service related + if self.provider.service_type: + kwargs['compute_service_type'] = self.provider.service_type + if self.provider.service_name: + kwargs['compute_service_name'] = self.provider.service_name + if self.provider.cloud is not None: + kwargs['cloud'] = self.provider.cloud + + auth_kwargs = {} + for auth_attr in ('username', 'password', 'auth_url'): + auth_val = getattr(self.provider, auth_attr) + if auth_val is not None: + auth_kwargs[auth_attr] = auth_val + + if self.provider.project_id is not None: + auth_kwargs['project_name'] = self.provider.project_id + + kwargs['auth'] = auth_kwargs + return shade.openstack_cloud(**kwargs) def runTask(self, task): try: @@ -454,6 +463,8 @@ class ProviderManager(TaskManager): return def waitForImage(self, image_id, timeout=3600): + # TODO(mordred): This should just be handled by the Fake, but we're + # not quite plumbed through for that yet if image_id == 'fake-glance-id': return True return self._waitForResource('image', image_id, timeout) @@ -496,37 +507,11 @@ class ProviderManager(TaskManager): def getImage(self, image_id): return self.submitTask(GetImageTask(image=image_id)) - def get_glance_client(self, provider): - keystone_kwargs = {'auth_url': provider.auth_url, - 'username': provider.username, - 'password': provider.password, - 'tenant_name': provider.project_id} - glance_kwargs = {'service_type': 'image'} - glance_endpoint_kwargs = {'service_type': 'image'} - - if provider.region_name: - keystone_kwargs['region_name'] = provider.region_name - glance_endpoint_kwargs['attr'] = 'region' - glance_endpoint_kwargs['filter_value'] = provider.region_name - - # get endpoint and authtoken - keystone = ksclient.Client(**keystone_kwargs) - glance_endpoint = keystone.service_catalog.url_for( - **glance_endpoint_kwargs) - glance_endpoint = glance_endpoint.replace('/v1.0', '') - - # configure glance client - glance = glanceclient.client.Client('1', glance_endpoint, - token=keystone.auth_token, - **glance_kwargs) - return glance - def uploadImage(self, image_name, filename, disk_format, container_format, meta): # configure glance and upload image. Note the meta flags # are provided as custom glance properties - glanceclient = self.get_glance_client(self.provider) - image = glanceclient.images.create( + image = self._client.glance_client.images.create( name=image_name, is_public=False, disk_format=disk_format, @@ -534,7 +519,6 @@ class ProviderManager(TaskManager): **meta) filename = '%s.%s' % (filename, disk_format) image.update(data=open(filename, 'rb')) - glanceclient = None return image.id def listExtensions(self): diff --git a/nodepool/tests/__init__.py b/nodepool/tests/__init__.py index afef79c4b..7937c8e02 100644 --- a/nodepool/tests/__init__.py +++ b/nodepool/tests/__init__.py @@ -39,7 +39,6 @@ class LoggingPopen(subprocess.Popen): class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase): - def setUp(self): super(BaseTestCase, self).setUp() test_timeout = os.environ.get('OS_TEST_TIMEOUT', 60) @@ -79,6 +78,8 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase): self.setUpFakes() def setUpFakes(self): + self.useFixture(fixtures.MonkeyPatch('shade.openstack_cloud', + fakeprovider.get_fake_client)) self.useFixture(fixtures.MonkeyPatch('keystoneclient.v2_0.client.' 'Client', fakeprovider.FakeKeystoneClient)) @@ -226,3 +227,8 @@ class DBTestCase(BaseTestCase): break time.sleep(1) self.wait_for_threads() + + +class IntegrationTestCase(DBTestCase): + def setUpFakes(self): + pass diff --git a/nodepool/tests/fixtures/config_validate/good.yaml b/nodepool/tests/fixtures/config_validate/good.yaml index c6b50faa8..19b9cf9e6 100644 --- a/nodepool/tests/fixtures/config_validate/good.yaml +++ b/nodepool/tests/fixtures/config_validate/good.yaml @@ -441,13 +441,8 @@ providers: username: jenkins private-key: /home/nodepool/.ssh/id_rsa - name: hpcloud-b1 + cloud: hpcloud region-name: 'region-b.geo-1' - service-type: 'compute' - service-name: 'Compute' - username: '<%= hpcloud_username %>' - password: '<%= hpcloud_password %>' - project-id: '<%= hpcloud_project %>' - auth-url: 'https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0' api-timeout: 60 boot-timeout: 120 max-servers: 100 diff --git a/nodepool/tests/fixtures/node_osc.yaml b/nodepool/tests/fixtures/node_osc.yaml new file mode 100644 index 000000000..225261a7f --- /dev/null +++ b/nodepool/tests/fixtures/node_osc.yaml @@ -0,0 +1,47 @@ +script-dir: . +dburi: '{dburi}' +images-dir: '{images_dir}' + +cron: + check: '*/15 * * * *' + cleanup: '*/1 * * * *' + image-update: '14 2 * * *' + +zmq-publishers: + - tcp://localhost:8881 + +#gearman-servers: +# - host: localhost + +labels: + - name: fake-label + image: fake-image + min-ready: 1 + providers: + - name: fake-provider + +providers: + - name: fake-provider + cloud: fake-cloud + keypair: 'if-present-use-this-keypair' + max-servers: 96 + pool: 'fake' + networks: + - net-id: 'some-uuid' + rate: 0.0001 + images: + - name: fake-image + base-image: 'Fake Precise' + min-ram: 8192 + name-filter: 'Fake' + meta: + key: value + key2: value + setup: prepare_node_devstack.sh + +targets: + - name: fake-target + jenkins: + url: https://jenkins.example.org/ + user: fake + apikey: fake diff --git a/nodepool/tests/test_nodepool.py b/nodepool/tests/test_nodepool.py index 7041aa19e..c8b4a5b03 100644 --- a/nodepool/tests/test_nodepool.py +++ b/nodepool/tests/test_nodepool.py @@ -299,7 +299,7 @@ class TestNodepool(tests.DBTestCase): # always raises a ProxyError. If our client reset works correctly # then we will create a new client object, which in this case would # be a new fake client in place of the bad client. - manager._client = nodepool.fakeprovider.BAD_CLIENT + manager._client = nodepool.fakeprovider.get_bad_client() # The only implemented function for the fake and bad clients # If we don't raise an uncaught exception, we pass @@ -308,7 +308,7 @@ class TestNodepool(tests.DBTestCase): # Now let's do it again, but let's prevent the client object from being # replaced and then assert that we raised the exception that we expect. manager._client = nodepool.fakeprovider.BAD_CLIENT - manager._getClient = lambda: nodepool.fakeprovider.BAD_CLIENT + manager._getClient = lambda: nodepool.fakeprovider.get_bad_client() with ExpectedException(requests.exceptions.ProxyError): manager.listExtensions() diff --git a/nodepool/tests/test_shade_integration.py b/nodepool/tests/test_shade_integration.py new file mode 100644 index 000000000..b5133c6b3 --- /dev/null +++ b/nodepool/tests/test_shade_integration.py @@ -0,0 +1,54 @@ +# Copyright (C) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import yaml + +from nodepool import tests + + +class TestShadeIntegration(tests.IntegrationTestCase): + def _cleanup_cloud_config(self): + os.remove('clouds.yaml') + + def _use_cloud_config(self, config): + with open('clouds.yaml', 'w') as h: + yaml.safe_dump(config, h) + self.addCleanup(self._cleanup_cloud_config) + + def test_nodepool_provider_config(self): + configfile = self.setup_config('node.yaml') + pool = self.useNodepool(configfile, watermark_sleep=1) + pool.updateConfig() + provider_manager = pool.config.provider_managers['fake-provider'] + auth_data = {'username': 'fake', + 'project_name': 'fake', + 'password': 'fake', + 'auth_url': 'fake'} + self.assertEqual(provider_manager._client.auth, auth_data) + + def test_nodepool_osc_config(self): + configfile = self.setup_config('node_osc.yaml') + auth_data = {'username': 'os_fake', + 'project_name': 'os_fake', + 'password': 'os_fake', + 'auth_url': 'os_fake'} + osc_config = {'clouds': {'fake-cloud': {'auth': auth_data}}} + self._use_cloud_config(osc_config) + + pool = self.useNodepool(configfile, watermark_sleep=1) + pool.updateConfig() + provider_manager = pool.config.provider_managers['fake-provider'] + self.assertEqual(provider_manager._client.auth, auth_data) diff --git a/requirements.txt b/requirements.txt index 39629dd80..811d46b22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,5 +16,7 @@ python-novaclient>=2.21.0 PyMySQL PrettyTable>=0.6,<0.8 six>=1.7.0 +# shade has a looser requirement on six than nodepool, so install six first +shade diskimage-builder voluptuous