zuul
/
nodepool
Code Issues Proposed changes

Use shade to get the client objects 

As an initial step in the transition to shade, just replace the getting
of the client objects with shade. This adds support for but does not
require using a clouds.yaml file and named clouds. This is an opt-in
feature and is not required by people using nodepool. But, having a
clouds.yaml on a server allows the same cloud configuration to also be
used by other tools, such as ansible and python-openstackclient, so
while not required, it's a nice to have.

Using shade to get the client connection handles also gives us
access to other client libraries, such as neutron, without structural
changes, although the attempt here is to not change anything logically
or move the ownership of any code tasks. The one exception is the
removal of using the novaclient extension for finding the neutron
networks, since shade neither has a facility to passing novaclient
extensions in as parameters, nor does it want to grow one.

Co-Authored By: Gregory Haynes <greg@greghaynes.net>

Depends-On: I22c33648e32cc7ce8fc163433b7c72912c28beb9
Change-Id: Iba84b8d578efa8cb7f5af85a5ffbc97f945a47c9
This commit is contained in:
Monty Taylor 2015-03-28 13:18:58 -04:00 committed by Gregory Haynes
parent 3c635ec9c2
commit 9e2937cedc
10 changed files with 203 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nodepool/cmd/config_validator.py
View File

@ -50,6 +50,7 @@ class ConfigValidator:
'service-name': str,
'availability-zones': [str],
'keypair': str,
'cloud': str,
'username': str,
'password': str,
'auth-url': str,

37
nodepool/fakeprovider.py
View File

@ -53,6 +53,26 @@ def get_fake_images_list():
return fake_images_list
BAD_CLIENT = None
def get_bad_client():
global BAD_CLIENT
if BAD_CLIENT is None:
BAD_CLIENT = BadOpenstackCloud()
return BAD_CLIENT
FAKE_CLIENT = None
def get_fake_client(**kwargs):
global FAKE_CLIENT
if FAKE_CLIENT is None:
FAKE_CLIENT = FakeOpenStackCloud()
return FAKE_CLIENT
class FakeList(object):
def __init__(self, l):
self._list = l
@ -143,9 +163,13 @@ class BadClient(FakeClient):
self.client = BadHTTPClient()
class BadOpenstackCloud(object):
nova_client = BadClient()
class FakeGlanceClient(object):
def __init__(self, *args, **kwargs):
self.id = 'fake-glance-id'
def __init__(self, **kwargs):
self.kwargs = kwargs
self.images = get_fake_images_list()
@ -160,6 +184,11 @@ class FakeKeystoneClient(object):
self.auth_token = 'fake-auth-token'
class FakeOpenStackCloud(object):
nova_client = FakeClient()
glance_client = FakeGlanceClient()
class FakeFile(StringIO.StringIO):
def __init__(self, path):
StringIO.StringIO.__init__(self)
@ -236,7 +265,3 @@ class FakeJenkins(object):
{u'name': u'test-view',
u'url': u'https://jenkins.example.com/view/test-view/'}]}
return d
FAKE_CLIENT = FakeClient()
BAD_CLIENT = BadClient()

10
nodepool/nodepool.py
View File

@ -1249,10 +1249,11 @@ class NodePool(threading.Thread):
p = Provider()
p.name = provider['name']
newconfig.providers[p.name] = p
p.username = provider['username']
p.password = provider['password']
p.project_id = provider['project-id']
p.auth_url = provider['auth-url']
p.username = provider.get('username')
p.password = provider.get('password')
p.project_id = provider.get('project-id')
p.auth_url = provider.get('auth-url')
p.cloud = provider.get('cloud')
p.service_type = provider.get('service-type')
p.service_name = provider.get('service-name')
p.region_name = provider.get('region-name')
@ -1397,6 +1398,7 @@ class NodePool(threading.Thread):
new_pm.password != old_pm.provider.password or
new_pm.project_id != old_pm.provider.project_id or
new_pm.auth_url != old_pm.provider.auth_url or
new_pm.cloud != old_pm.provider.cloud or
new_pm.service_type != old_pm.provider.service_type or
new_pm.service_name != old_pm.provider.service_name or
new_pm.max_servers != old_pm.provider.max_servers or

120
nodepool/provider_manager.py
View File

@ -18,18 +18,15 @@
import logging
import paramiko
import novaclient
import novaclient.client
import novaclient.extension
import novaclient.v2.contrib.tenant_networks
import threading
import glanceclient
import glanceclient.client
import keystoneclient.v2_0.client as ksclient
import time
import requests.exceptions
import sys
import shade
import novaclient
from nodeutils import iterate_timeout
from task_manager import Task, TaskManager, ManagerStoppedException
@ -113,14 +110,14 @@ class NotFound(Exception):
class CreateServerTask(Task):
def main(self, client):
server = client.servers.create(**self.args)
server = client.nova_client.servers.create(**self.args)
return str(server.id)
class GetServerTask(Task):
def main(self, client):
try:
server = client.servers.get(self.args['server_id'])
server = client.nova_client.servers.get(self.args['server_id'])
except novaclient.exceptions.NotFound:
raise NotFound()
return make_server_dict(server)
@ -128,52 +125,52 @@ class GetServerTask(Task):
class DeleteServerTask(Task):
def main(self, client):
client.servers.delete(self.args['server_id'])
client.nova_client.servers.delete(self.args['server_id'])
class ListServersTask(Task):
def main(self, client):
servers = client.servers.list()
servers = client.nova_client.servers.list()
return [make_server_dict(server) for server in servers]
class AddKeypairTask(Task):
def main(self, client):
client.keypairs.create(**self.args)
client.nova_client.keypairs.create(**self.args)
class ListKeypairsTask(Task):
def main(self, client):
keys = client.keypairs.list()
keys = client.nova_client.keypairs.list()
return [dict(id=str(key.id), name=key.name) for
key in keys]
class DeleteKeypairTask(Task):
def main(self, client):
client.keypairs.delete(self.args['name'])
client.nova_client.keypairs.delete(self.args['name'])
class CreateFloatingIPTask(Task):
def main(self, client):
ip = client.floating_ips.create(**self.args)
ip = client.nova_client.floating_ips.create(**self.args)
return dict(id=str(ip.id), ip=ip.ip)
class AddFloatingIPTask(Task):
def main(self, client):
client.servers.add_floating_ip(**self.args)
client.nova_client.servers.add_floating_ip(**self.args)
class GetFloatingIPTask(Task):
def main(self, client):
ip = client.floating_ips.get(self.args['ip_id'])
ip = client.nova_client.floating_ips.get(self.args['ip_id'])
return dict(id=str(ip.id), ip=ip.ip, instance_id=str(ip.instance_id))
class ListFloatingIPsTask(Task):
def main(self, client):
ips = client.floating_ips.list()
ips = client.nova_client.floating_ips.list()
return [dict(id=str(ip.id), ip=ip.ip,
instance_id=str(ip.instance_id)) for
ip in ips]
@ -181,24 +178,24 @@ class ListFloatingIPsTask(Task):
class RemoveFloatingIPTask(Task):
def main(self, client):
client.servers.remove_floating_ip(**self.args)
client.nova_client.servers.remove_floating_ip(**self.args)
class DeleteFloatingIPTask(Task):
def main(self, client):
client.floating_ips.delete(self.args['ip_id'])
client.nova_client.floating_ips.delete(self.args['ip_id'])
class CreateImageTask(Task):
def main(self, client):
# This returns an id
return str(client.servers.create_image(**self.args))
return str(client.nova_client.servers.create_image(**self.args))
class GetImageTask(Task):
def main(self, client):
try:
image = client.images.get(**self.args)
image = client.nova_client.images.get(**self.args)
except novaclient.exceptions.NotFound:
raise NotFound()
# HP returns 404, rackspace can return a 'DELETED' image.
@ -210,7 +207,7 @@ class GetImageTask(Task):
class ListExtensionsTask(Task):
def main(self, client):
try:
resp, body = client.client.get('/extensions')
resp, body = client.nova_client.client.get('/extensions')
return [x['alias'] for x in body['extensions']]
except novaclient.exceptions.NotFound:
# No extensions present.
@ -219,32 +216,33 @@ class ListExtensionsTask(Task):
class ListFlavorsTask(Task):
def main(self, client):
flavors = client.flavors.list()
flavors = client.nova_client.flavors.list()
return [dict(id=str(flavor.id), ram=flavor.ram, name=flavor.name)
for flavor in flavors]
class ListImagesTask(Task):
def main(self, client):
images = client.images.list()
images = client.nova_client.images.list()
return [make_image_dict(image) for image in images]
class FindImageTask(Task):
def main(self, client):
image = client.images.find(**self.args)
image = client.nova_client.images.find(**self.args)
return dict(id=str(image.id))
class DeleteImageTask(Task):
def main(self, client):
client.images.delete(**self.args)
client.nova_client.images.delete(**self.args)
class FindNetworkTask(Task):
def main(self, client):
network = client.tenant_networks.find(**self.args)
return dict(id=str(network.id))
for network in client.neutron_client.list_networks()['networks']:
if self.args['label'] == network['name']:
return dict(id=str(network['id']))
class ProviderManager(TaskManager):
@ -285,20 +283,31 @@ class ProviderManager(TaskManager):
self._cloud_metadata_read = True
def _getClient(self):
tenant_networks = novaclient.extension.Extension(
'tenant_networks', novaclient.v2.contrib.tenant_networks)
args = ['1.1', self.provider.username, self.provider.password,
self.provider.project_id, self.provider.auth_url]
kwargs = {'extensions': [tenant_networks]}
if self.provider.service_type:
kwargs['service_type'] = self.provider.service_type
if self.provider.service_name:
kwargs['service_name'] = self.provider.service_name
kwargs = {}
if self.provider.region_name:
kwargs['region_name'] = self.provider.region_name
if self.provider.api_timeout:
kwargs['timeout'] = self.provider.api_timeout
return novaclient.client.Client(*args, **kwargs)
kwargs['api_timeout'] = self.provider.api_timeout
# These are named from back when we only talked to Nova. They're
# actually compute service related
if self.provider.service_type:
kwargs['compute_service_type'] = self.provider.service_type
if self.provider.service_name:
kwargs['compute_service_name'] = self.provider.service_name
if self.provider.cloud is not None:
kwargs['cloud'] = self.provider.cloud
auth_kwargs = {}
for auth_attr in ('username', 'password', 'auth_url'):
auth_val = getattr(self.provider, auth_attr)
if auth_val is not None:
auth_kwargs[auth_attr] = auth_val
if self.provider.project_id is not None:
auth_kwargs['project_name'] = self.provider.project_id
kwargs['auth'] = auth_kwargs
return shade.openstack_cloud(**kwargs)
def runTask(self, task):
try:
@ -454,6 +463,8 @@ class ProviderManager(TaskManager):
return
def waitForImage(self, image_id, timeout=3600):
# TODO(mordred): This should just be handled by the Fake, but we're
# not quite plumbed through for that yet
if image_id == 'fake-glance-id':
return True
return self._waitForResource('image', image_id, timeout)
@ -496,37 +507,11 @@ class ProviderManager(TaskManager):
def getImage(self, image_id):
return self.submitTask(GetImageTask(image=image_id))
def get_glance_client(self, provider):
keystone_kwargs = {'auth_url': provider.auth_url,
'username': provider.username,
'password': provider.password,
'tenant_name': provider.project_id}
glance_kwargs = {'service_type': 'image'}
glance_endpoint_kwargs = {'service_type': 'image'}
if provider.region_name:
keystone_kwargs['region_name'] = provider.region_name
glance_endpoint_kwargs['attr'] = 'region'
glance_endpoint_kwargs['filter_value'] = provider.region_name
# get endpoint and authtoken
keystone = ksclient.Client(**keystone_kwargs)
glance_endpoint = keystone.service_catalog.url_for(
**glance_endpoint_kwargs)
glance_endpoint = glance_endpoint.replace('/v1.0', '')
# configure glance client
glance = glanceclient.client.Client('1', glance_endpoint,
token=keystone.auth_token,
**glance_kwargs)
return glance
def uploadImage(self, image_name, filename, disk_format, container_format,
meta):
# configure glance and upload image. Note the meta flags
# are provided as custom glance properties
glanceclient = self.get_glance_client(self.provider)
image = glanceclient.images.create(
image = self._client.glance_client.images.create(
name=image_name,
is_public=False,
disk_format=disk_format,
@ -534,7 +519,6 @@ class ProviderManager(TaskManager):
**meta)
filename = '%s.%s' % (filename, disk_format)
image.update(data=open(filename, 'rb'))
glanceclient = None
return image.id
def listExtensions(self):

8
nodepool/tests/__init__.py
View File

@ -39,7 +39,6 @@ class LoggingPopen(subprocess.Popen):
class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
def setUp(self):
super(BaseTestCase, self).setUp()
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 60)
@ -79,6 +78,8 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
self.setUpFakes()
def setUpFakes(self):
self.useFixture(fixtures.MonkeyPatch('shade.openstack_cloud',
fakeprovider.get_fake_client))
self.useFixture(fixtures.MonkeyPatch('keystoneclient.v2_0.client.'
'Client',
fakeprovider.FakeKeystoneClient))
@ -226,3 +227,8 @@ class DBTestCase(BaseTestCase):
break
time.sleep(1)
self.wait_for_threads()
class IntegrationTestCase(DBTestCase):
def setUpFakes(self):
pass

7
nodepool/tests/fixtures/config_validate/good.yaml vendored
View File

@ -441,13 +441,8 @@ providers:
username: jenkins
private-key: /home/nodepool/.ssh/id_rsa
- name: hpcloud-b1
cloud: hpcloud
region-name: 'region-b.geo-1'
service-type: 'compute'
service-name: 'Compute'
username: '<%= hpcloud_username %>'
password: '<%= hpcloud_password %>'
project-id: '<%= hpcloud_project %>'
auth-url: 'https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0'
api-timeout: 60
boot-timeout: 120
max-servers: 100

47
nodepool/tests/fixtures/node_osc.yaml vendored Normal file
View File

@ -0,0 +1,47 @@
script-dir: .
dburi: '{dburi}'
images-dir: '{images_dir}'
cron:
check: '*/15 * * * *'
cleanup: '*/1 * * * *'
image-update: '14 2 * * *'
zmq-publishers:
- tcp://localhost:8881
#gearman-servers:
# - host: localhost
labels:
- name: fake-label
image: fake-image
min-ready: 1
providers:
- name: fake-provider
providers:
- name: fake-provider
cloud: fake-cloud
keypair: 'if-present-use-this-keypair'
max-servers: 96
pool: 'fake'
networks:
- net-id: 'some-uuid'
rate: 0.0001
images:
- name: fake-image
base-image: 'Fake Precise'
min-ram: 8192
name-filter: 'Fake'
meta:
key: value
key2: value
setup: prepare_node_devstack.sh
targets:
- name: fake-target
jenkins:
url: https://jenkins.example.org/
user: fake
apikey: fake

4
nodepool/tests/test_nodepool.py
View File

@ -299,7 +299,7 @@ class TestNodepool(tests.DBTestCase):
# always raises a ProxyError. If our client reset works correctly
# then we will create a new client object, which in this case would
# be a new fake client in place of the bad client.
manager._client = nodepool.fakeprovider.BAD_CLIENT
manager._client = nodepool.fakeprovider.get_bad_client()
# The only implemented function for the fake and bad clients
# If we don't raise an uncaught exception, we pass
@ -308,7 +308,7 @@ class TestNodepool(tests.DBTestCase):
# Now let's do it again, but let's prevent the client object from being
# replaced and then assert that we raised the exception that we expect.
manager._client = nodepool.fakeprovider.BAD_CLIENT
manager._getClient = lambda: nodepool.fakeprovider.BAD_CLIENT
manager._getClient = lambda: nodepool.fakeprovider.get_bad_client()
with ExpectedException(requests.exceptions.ProxyError):
manager.listExtensions()

54
nodepool/tests/test_shade_integration.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright (C) 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import yaml
from nodepool import tests
class TestShadeIntegration(tests.IntegrationTestCase):
def _cleanup_cloud_config(self):
os.remove('clouds.yaml')
def _use_cloud_config(self, config):
with open('clouds.yaml', 'w') as h:
yaml.safe_dump(config, h)
self.addCleanup(self._cleanup_cloud_config)
def test_nodepool_provider_config(self):
configfile = self.setup_config('node.yaml')
pool = self.useNodepool(configfile, watermark_sleep=1)
pool.updateConfig()
provider_manager = pool.config.provider_managers['fake-provider']
auth_data = {'username': 'fake',
'project_name': 'fake',
'password': 'fake',
'auth_url': 'fake'}
self.assertEqual(provider_manager._client.auth, auth_data)
def test_nodepool_osc_config(self):
configfile = self.setup_config('node_osc.yaml')
auth_data = {'username': 'os_fake',
'project_name': 'os_fake',
'password': 'os_fake',
'auth_url': 'os_fake'}
osc_config = {'clouds': {'fake-cloud': {'auth': auth_data}}}
self._use_cloud_config(osc_config)
pool = self.useNodepool(configfile, watermark_sleep=1)
pool.updateConfig()
provider_manager = pool.config.provider_managers['fake-provider']
self.assertEqual(provider_manager._client.auth, auth_data)

2
requirements.txt
View File

@ -16,5 +16,7 @@ python-novaclient>=2.21.0
PyMySQL
PrettyTable>=0.6,<0.8
six>=1.7.0
# shade has a looser requirement on six than nodepool, so install six first
shade
diskimage-builder
voluptuous