From 2e05f1850f811efbd1a802c06f8de3cdabd7cc3b Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 5 Oct 2015 10:50:54 -0700 Subject: [PATCH] Restore ability to run nodepoold with fakes With OSC and shade patches, we lost the ability to run nodepoold in the foreground with fakes. This restores that ability. The shade integration unit tests are updated to use the string 'real' rather than 'fake' in config files, as they are trying to avoid actually using the nodepool fakes, and the use of the string 'fake' is what triggers their use in many cases. Change-Id: Ia5d3c3d5462bc03edafcc1567d1bab299ea5d40f --- nodepool/config.py | 22 ++++++++ nodepool/fakeprovider.py | 5 +- nodepool/provider_manager.py | 20 +++++++- nodepool/tests/fixtures/integration.yaml | 51 +++++++++++++++++++ .../{node_osc.yaml => integration_osc.yaml} | 18 +++---- nodepool/tests/test_shade_integration.py | 46 ++++++++--------- tools/fake.yaml | 1 + 7 files changed, 128 insertions(+), 35 deletions(-) create mode 100644 nodepool/tests/fixtures/integration.yaml rename nodepool/tests/fixtures/{node_osc.yaml => integration_osc.yaml} (71%) diff --git a/nodepool/config.py b/nodepool/config.py index 66a3f24f7..54c012c88 100644 --- a/nodepool/config.py +++ b/nodepool/config.py @@ -1,7 +1,27 @@ +#!/usr/bin/env python + +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + import os_client_config from six.moves import configparser as ConfigParser import yaml +import fakeprovider + class ConfigValue(object): def __eq__(self, other): @@ -323,4 +343,6 @@ def _cloudKwargsFromProvider(provider): def _get_one_cloud(cloud_config, cloud_kwargs): '''This is a function to allow for overriding it in tests.''' + if cloud_kwargs.get('auth', {}).get('auth-url', '') == 'fake': + return fakeprovider.fake_get_one_cloud(cloud_config, cloud_kwargs) return cloud_config.get_one_cloud(**cloud_kwargs) diff --git a/nodepool/fakeprovider.py b/nodepool/fakeprovider.py index 4fd486f37..337354ef1 100644 --- a/nodepool/fakeprovider.py +++ b/nodepool/fakeprovider.py @@ -66,6 +66,7 @@ class Dummy(object): def fake_get_one_cloud(cloud_config, cloud_kwargs): cloud_kwargs['validate'] = False + cloud_kwargs['image_format'] = 'qcow2' return cloud_config.get_one_cloud(**cloud_kwargs) @@ -184,11 +185,11 @@ class FakeOpenStackCloud(object): def delete_image(self, name_or_id): self._delete(name_or_id, self._image_list) - def create_image_snapshot(self, server, image_name, **metadata): + def create_image_snapshot(self, name, server, **metadata): # XXX : validate metadata? return self._create( self._image_list, instance_type=Dummy.IMAGE, - name=image_name, **metadata) + name=name, **metadata) def list_flavors(self, get_extra=False): return self._flavor_list diff --git a/nodepool/provider_manager.py b/nodepool/provider_manager.py index bb54ed758..a54234301 100644 --- a/nodepool/provider_manager.py +++ b/nodepool/provider_manager.py @@ -24,6 +24,7 @@ from contextlib import contextmanager import shade import exceptions +import fakeprovider from nodeutils import iterate_timeout from task_manager import TaskManager, ManagerStoppedException @@ -52,6 +53,13 @@ class NotFound(Exception): pass +def get_provider_manager(provider): + if (provider.cloud_config.get_auth_args().get('auth_url') == 'fake'): + return FakeProviderManager(provider) + else: + return ProviderManager(provider) + + class ProviderManager(TaskManager): log = logging.getLogger("nodepool.ProviderManager") @@ -70,7 +78,8 @@ class ProviderManager(TaskManager): else: ProviderManager.log.debug("Creating new ProviderManager object" " for %s" % p.name) - new_config.provider_managers[p.name] = ProviderManager(p) + new_config.provider_managers[p.name] = \ + get_provider_manager(p) new_config.provider_managers[p.name].start() for stop_manager in stop_managers: @@ -326,3 +335,12 @@ class ProviderManager(TaskManager): self.log.debug('Deleting server %s' % server_id) self.deleteServer(server_id) + + +class FakeProviderManager(ProviderManager): + def __init__(self, provider): + self.__client = fakeprovider.FakeOpenStackCloud() + super(FakeProviderManager, self).__init__(provider) + + def _getClient(self): + return self.__client diff --git a/nodepool/tests/fixtures/integration.yaml b/nodepool/tests/fixtures/integration.yaml new file mode 100644 index 000000000..18180f783 --- /dev/null +++ b/nodepool/tests/fixtures/integration.yaml @@ -0,0 +1,51 @@ +script-dir: . +images-dir: '{images_dir}' + +cron: + check: '*/15 * * * *' + cleanup: '*/1 * * * *' + image-update: '14 2 * * *' + +zmq-publishers: + - tcp://localhost:8881 + +gearman-servers: + - host: localhost + port: {gearman_port} + +labels: + - name: real-label + image: real-image + min-ready: 1 + providers: + - name: real-provider + +providers: + - name: real-provider + region-name: real-region + keypair: 'if-present-use-this-keypair' + username: 'real' + password: 'real' + auth-url: 'real' + project-id: 'real' + max-servers: 96 + pool: 'real' + networks: + - net-id: 'some-uuid' + rate: 0.0001 + images: + - name: real-image + base-image: 'Real Precise' + min-ram: 8192 + name-filter: 'Real' + meta: + key: value + key2: value + setup: prepare_node_devstack.sh + +targets: + - name: fake-target + jenkins: + url: https://jenkins.example.org/ + user: fake + apikey: fake diff --git a/nodepool/tests/fixtures/node_osc.yaml b/nodepool/tests/fixtures/integration_osc.yaml similarity index 71% rename from nodepool/tests/fixtures/node_osc.yaml rename to nodepool/tests/fixtures/integration_osc.yaml index 624c48ca2..ffcbb5ad3 100644 --- a/nodepool/tests/fixtures/node_osc.yaml +++ b/nodepool/tests/fixtures/integration_osc.yaml @@ -14,26 +14,26 @@ gearman-servers: port: {gearman_port} labels: - - name: fake-label - image: fake-image + - name: real-label + image: real-image min-ready: 1 providers: - - name: fake-provider + - name: real-provider providers: - - name: fake-provider - cloud: fake-cloud + - name: real-provider + cloud: real-cloud keypair: 'if-present-use-this-keypair' max-servers: 96 - pool: 'fake' + pool: 'real' networks: - net-id: 'some-uuid' rate: 0.0001 images: - - name: fake-image - base-image: 'Fake Precise' + - name: real-image + base-image: 'Real Precise' min-ram: 8192 - name-filter: 'Fake' + name-filter: 'Real' meta: key: value key2: value diff --git a/nodepool/tests/test_shade_integration.py b/nodepool/tests/test_shade_integration.py index 1c4cfb14e..b1430d20a 100644 --- a/nodepool/tests/test_shade_integration.py +++ b/nodepool/tests/test_shade_integration.py @@ -42,53 +42,53 @@ class TestShadeIntegration(tests.IntegrationTestCase): self.addCleanup(self._cleanup_cloud_config) def test_nodepool_provider_config(self): - configfile = self.setup_config('node.yaml') + configfile = self.setup_config('integration.yaml') pool = self.useNodepool(configfile, watermark_sleep=1) pool.updateConfig() - provider_manager = pool.config.provider_managers['fake-provider'] - auth_data = {'username': 'fake', - 'project_id': 'fake', - 'password': 'fake', - 'auth_url': 'fake'} + provider_manager = pool.config.provider_managers['real-provider'] + auth_data = {'username': 'real', + 'project_id': 'real', + 'password': 'real', + 'auth_url': 'real'} self.assertEqual(provider_manager._client.auth, auth_data) - self.assertEqual(provider_manager._client.region_name, 'fake-region') + self.assertEqual(provider_manager._client.region_name, 'real-region') def test_nodepool_osc_config(self): - configfile = self.setup_config('node_osc.yaml') - auth_data = {'username': 'os_fake', - 'project_name': 'os_fake', - 'password': 'os_fake', - 'auth_url': 'os_fake'} - osc_config = {'clouds': {'fake-cloud': {'auth': auth_data}}} + configfile = self.setup_config('integration_osc.yaml') + auth_data = {'username': 'os_real', + 'project_name': 'os_real', + 'password': 'os_real', + 'auth_url': 'os_real'} + osc_config = {'clouds': {'real-cloud': {'auth': auth_data}}} self._use_cloud_config(osc_config) pool = self.useNodepool(configfile, watermark_sleep=1) pool.updateConfig() - provider_manager = pool.config.provider_managers['fake-provider'] + provider_manager = pool.config.provider_managers['real-provider'] self.assertEqual(provider_manager._client.auth, auth_data) def test_nodepool_osc_config_reload(self): - configfile = self.setup_config('node_osc.yaml') - auth_data = {'username': 'os_fake', - 'project_name': 'os_fake', - 'password': 'os_fake', - 'auth_url': 'os_fake'} - osc_config = {'clouds': {'fake-cloud': {'auth': auth_data}}} + configfile = self.setup_config('integration_osc.yaml') + auth_data = {'username': 'os_real', + 'project_name': 'os_real', + 'password': 'os_real', + 'auth_url': 'os_real'} + osc_config = {'clouds': {'real-cloud': {'auth': auth_data}}} self._use_cloud_config(osc_config) pool = self.useNodepool(configfile, watermark_sleep=1) pool.updateConfig() - provider_manager = pool.config.provider_managers['fake-provider'] + provider_manager = pool.config.provider_managers['real-provider'] self.assertEqual(provider_manager._client.auth, auth_data) # update the config - auth_data['password'] = 'os_new_fake' + auth_data['password'] = 'os_new_real' os.remove(self.clouds_path) with open(self.clouds_path, 'w') as h: yaml.safe_dump(osc_config, h) pool.updateConfig() - provider_manager = pool.config.provider_managers['fake-provider'] + provider_manager = pool.config.provider_managers['real-provider'] self.assertEqual(provider_manager._client.auth, auth_data) def test_exceptions(self): diff --git a/tools/fake.yaml b/tools/fake.yaml index 2079e0332..c73f0084a 100644 --- a/tools/fake.yaml +++ b/tools/fake.yaml @@ -1,4 +1,5 @@ script-dir: . +images-dir: tools/images cron: check: '*/15 * * * *'