openstack/fuel-web
Code Issues Proposed changes

Always send versioned paths to manifests/repos

Browse Source 
Since Fuel 6.0 we always have versioned folders for puppet manifests and
repos, so there no need to send unversioned paths by default. The patch
implements it by adding records to release_orchestrator_data for initial
releases.

Change-Id: Icb7c4449696e4032e6d8e1e939ae621614e2bc18
Closes-Bug: #1395047
This commit is contained in:
Igor Kalnitsky 2014-11-24 15:15:09 +02:00
parent 20890bfb5b
commit a1c1d3f2f9
11 changed files with 184 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nailgun/nailgun/api/v1/validators/release.py
View File

@ -35,6 +35,7 @@ class ReleaseValidator(BasicValidator):
"Invalid network data: {0}".format(network),
log_message=True
)
if "orchestrator_data" in d:
if not isinstance(d["orchestrator_data"], dict):
raise errors.InvalidData(
@ -68,6 +69,11 @@ class ReleaseValidator(BasicValidator):
"No release operating system specified",
log_message=True
)
if "orchestrator_data" not in d:
raise errors.InvalidData(
'No orchestrator_data specified', log_message=True
)
if db().query(Release).filter_by(
name=d["name"],
version=d["version"]

14
nailgun/nailgun/db/sqlalchemy/fixman.py
View File

@ -106,8 +106,20 @@ def upload_fixture(fileobj, loader=None):
except Exception:
break
new_obj = obj['model']()
# NOTE(ikalnitsly):
# In order to add a release to Nailgun we have to fill two tables:
# releases and release_orchestrator_data. By using former fixture
# approach we can't do it, since the fixture is bond to only one
# database model and can't deal with additional logic. Therefore
# we need to use Nailgun's objects which know how to handle it.
#
# TODO(ikalnitsky):
# Rewrite fixture logic - it must be simple and obvious.
if obj['model'] is objects.Release.model:
objects.Release.create(obj['fields'])
continue
new_obj = obj['model']()
fk_fields = {}
for field, value in obj["fields"].iteritems():
f = getattr(obj['model'], field)

10
nailgun/nailgun/fixtures/openstack.yaml
View File

@ -1218,6 +1218,11 @@
uri: "http://{{settings.MASTER_IP}}:8080/targetimages/centos_65_x86_64-boot.img.gz"
format: "ext2"
container: "gzip"
orchestrator_data:
puppet_manifests_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/"
puppet_modules_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/"
repo_metadata:
"{OPENSTACK_VERSION}": "http://{MASTER_IP}:8080/{OPENSTACK_VERSION}/centos/x86_64"
- pk: 2
extend: *base_release
fields:
@ -1241,3 +1246,8 @@
uri: "http://{{settings.MASTER_IP}}:8080/targetimages/ubuntu_1204_amd64-boot.img.gz"
format: "ext2"
container: "gzip"
orchestrator_data:
puppet_manifests_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/"
puppet_modules_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/"
repo_metadata:
"{OPENSTACK_VERSION}": "http://{MASTER_IP}:8080/{OPENSTACK_VERSION}/ubuntu/x86_64 precise main"

55
nailgun/nailgun/objects/release.py
View File

@ -24,7 +24,7 @@ from sqlalchemy import not_
from nailgun import consts
from nailgun.objects.serializers.release import ReleaseSerializer
from nailgun.objects.serializers import release as release_serializer
from nailgun.db import db
@ -43,6 +43,9 @@ class ReleaseOrchestratorData(NailgunObject):
#: SQLAlchemy model
model = models.ReleaseOrchestratorData
#: Serializer for ReleaseOrchestratorData
serializer = release_serializer.ReleaseOrchestratorDataSerializer
#: JSON schema
schema = {
"$schema": "http://json-schema.org/draft-04/schema#",
@ -84,13 +87,14 @@ class ReleaseOrchestratorData(NailgunObject):
release = Release.get_by_uid(rendered_data['release_id'])
context = {
'MASTER_IP': settings.MASTER_IP,
'OPENSTACK_VERSION': release.version,
}
'OPENSTACK_VERSION': release.version}
# render all the paths
repo_metadata = {}
for key, value in six.iteritems(rendered_data['repo_metadata']):
rendered_data['repo_metadata'][key] = \
cls.render_path(value, context)
formatted_key = cls.render_path(key, context)
repo_metadata[formatted_key] = cls.render_path(value, context)
rendered_data['repo_metadata'] = repo_metadata
rendered_data['puppet_manifests_source'] = \
cls.render_path(rendered_data.get(
@ -115,7 +119,7 @@ class Release(NailgunObject):
model = models.Release
#: Serializer for Release
serializer = ReleaseSerializer
serializer = release_serializer.ReleaseSerializer
#: Release JSON schema
schema = {
@ -220,41 +224,16 @@ class Release(NailgunObject):
@classmethod
def update_orchestrator_data(cls, instance, orchestrator_data):
for k in ["id", "release_id"]:
orchestrator_data.pop(k, None)
if orchestrator_data:
if instance.orchestrator_data:
ReleaseOrchestratorData.update(
instance.orchestrator_data, orchestrator_data)
else:
orchestrator_data["release_id"] = instance.id
ReleaseOrchestratorData.create(orchestrator_data)
orchestrator_data.pop("id", None)
orchestrator_data["release_id"] = instance.id
ReleaseOrchestratorData.update(
instance.orchestrator_data, orchestrator_data)
@classmethod
def get_orchestrator_data_dict(cls, instance):
os = instance.operating_system.lower()
default_orchestrator_data = {
"repo_metadata": {
"nailgun":
settings.DEFAULT_REPO[os].format(
MASTER_IP=settings.MASTER_IP),
},
"puppet_modules_source":
settings.DEFAULT_PUPPET['modules'].format(
MASTER_IP=settings.MASTER_IP),
"puppet_manifests_source":
settings.DEFAULT_PUPPET['manifests'].format(
MASTER_IP=settings.MASTER_IP),
}
return {
"repo_metadata":
instance.orchestrator_data.repo_metadata,
"puppet_modules_source":
instance.orchestrator_data.puppet_modules_source,
"puppet_manifests_source":
instance.orchestrator_data.puppet_manifests_source
} if instance.orchestrator_data else default_orchestrator_data
data = instance.orchestrator_data
return ReleaseOrchestratorData.serializer.serialize(data)
@classmethod
def is_deployable(cls, instance):

7
nailgun/nailgun/settings.yaml
View File

@ -62,13 +62,6 @@ RABBITMQ:
fake: "0"
hostname: "127.0.0.1"
DEFAULT_PUPPET:
modules: "rsync://{MASTER_IP}:/puppet/modules/"
manifests: "rsync://{MASTER_IP}:/puppet/manifests/"
DEFAULT_REPO:
centos: "http://{MASTER_IP}:8080/centos/x86_64"
ubuntu: "http://{MASTER_IP}:8080/ubuntu/x86_64 precise main"
PLUGINS_PATH: '/var/www/nailgun/plugins'
PLUGINS_SLAVES_SCRIPTS_PATH: '/etc/fuel/plugins/{plugin_name}/'
PLUGINS_REPO_URL: 'http://{master_ip}:8080/plugins/{plugin_name}/'

16
nailgun/nailgun/test/base.py
View File

@ -134,6 +134,7 @@ class Environment(object):
'attributes_metadata': self.get_default_attributes_metadata(),
'volumes_metadata': self.get_default_volumes_metadata(),
'roles_metadata': self.get_default_roles_metadata(),
'orchestrator_data': self.get_default_orchestrator_data(),
}
if kwargs:
release_data.update(kwargs)
@ -469,6 +470,21 @@ class Environment(object):
sample_plugin.update(kwargs)
return sample_plugin
def get_default_orchestrator_data(self, **kwargs):
orchestrator_data = {
'puppet_manifests_source':
'rsync://127.0.0.1:/puppet/2014.2-6.0/manifests/',
'puppet_modules_source':
'rsync://127.0.0.1:/puppet/2014.2-6.0/modules/',
'repo_metadata': {
'2014.2-6.0': 'http://127.0.0.1:8080/2014.2-6.0/centos/x86_64'
}
}
orchestrator_data.update(kwargs)
return orchestrator_data
def upload_fixtures(self, fxtr_names):
for fxtr_path in self.fxtr_paths_by_names(fxtr_names):
with open(fxtr_path, "r") as fxtr_file:

6
nailgun/nailgun/test/integration/test_orchestrator_serializer.py
View File

@ -1529,16 +1529,16 @@ class TestRepoAndPuppetDataSerialization(OrchestratorSerializerTestBase):
self.assertEqual(
fact['repo_metadata'],
{
'nailgun': 'http://127.0.0.1:8080/centos/x86_64'
'2014.2-6.0': 'http://127.0.0.1:8080/2014.2-6.0/centos/x86_64'
}
)
self.assertEqual(
fact['puppet_modules_source'],
'rsync://127.0.0.1:/puppet/modules/'
'rsync://127.0.0.1:/puppet/2014.2-6.0/modules/'
)
self.assertEqual(
fact['puppet_manifests_source'],
'rsync://127.0.0.1:/puppet/manifests/'
'rsync://127.0.0.1:/puppet/2014.2-6.0/manifests/'
)
def test_orch_data_w_replaced_deployment_info(self):

6
nailgun/nailgun/test/unit/test_attributes_plugin.py
View File

@ -33,8 +33,10 @@ class TestPlugin(base.BaseTestCase):
self.plugin = Plugin.create(self.plugin_metadata)
self.env.create(
cluster_kwargs={'mode': 'multinode'},
release_kwargs={'version': '2014.2-6.0',
'operating_system': 'Ubuntu'})
release_kwargs={
'version': '2014.2-6.0',
'operating_system': 'Ubuntu',
'orchestrator_data': self.env.get_default_orchestrator_data()})
self.cluster = self.env.clusters[0]
self.attr_plugin = attr_plugin.ClusterAttributesPlugin(self.plugin)
self.env_config = self.env.get_default_plugin_env_config()

6
nailgun/nailgun/test/unit/test_objects.py
View File

@ -687,7 +687,8 @@ class TestReleaseOrchestratorData(BaseIntegrationTest):
self.data = {
'release_id': self.release.id,
'repo_metadata': {
'5': 'http://10.20.0.2:8080/{OPENSTACK_VERSION}/centos/x86_64',
'{OPENSTACK_VERSION}':
'http://10.20.0.2:8080/{OPENSTACK_VERSION}/centos/x86_64',
},
'puppet_manifests_source': 'rsync://10.20.0.2:/puppet/modules/',
'puppet_modules_source': 'rsync://10.20.0.2:/puppet/manifests/',
@ -695,7 +696,8 @@ class TestReleaseOrchestratorData(BaseIntegrationTest):
instance = objects.ReleaseOrchestratorData.create(self.data)
self.assertEqual(instance.repo_metadata, {
'5': 'http://10.20.0.2:8080/{0}/centos/x86_64'.format(
self.release.version:
'http://10.20.0.2:8080/{0}/centos/x86_64'.format(
self.release.version)})
self.assertEqual(
instance.puppet_manifests_source,

20
nailgun/nailgun/test/unit/test_release_collection_handlers.py
View File

@ -36,7 +36,9 @@ class TestHandlers(BaseIntegrationTest):
params=jsonutils.dumps({
'name': 'Another test release',
'version': '1.0',
'operating_system': 'CentOS'
'operating_system': 'CentOS',
'orchestrator_data':
self.env.get_default_orchestrator_data(),
}),
headers=self.default_headers
)
@ -80,7 +82,9 @@ class TestHandlers(BaseIntegrationTest):
}
]
}
}
},
'orchestrator_data':
self.env.get_default_orchestrator_data(),
}),
headers=self.default_headers
)
@ -109,7 +113,9 @@ class TestHandlers(BaseIntegrationTest):
}
]
}
}
},
'orchestrator_data':
self.env.get_default_orchestrator_data()
}),
headers=self.default_headers,
expect_errors=True
@ -161,7 +167,9 @@ class TestHandlers(BaseIntegrationTest):
}
]
}
}
},
'orchestrator_data':
self.env.get_default_orchestrator_data()
}),
headers=self.default_headers
)
@ -190,7 +198,9 @@ class TestHandlers(BaseIntegrationTest):
}
]
}
}
},
'orchestrator_data':
self.env.get_default_orchestrator_data(),
}),
headers=self.default_headers,
expect_errors=True

96
nailgun/nailgun/test/unit/test_release_validator.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from nailgun.openstack.common import jsonutils
from nailgun.api.v1.validators.release import ReleaseValidator
from nailgun.errors import errors
from nailgun.test.base import BaseTestCase
class TestReleaseValidator(BaseTestCase):
def setUp(self):
super(TestReleaseValidator, self).setUp()
self.release = {
'name': 'Test Release',
'version': '2014.2-6.0',
'operating_system': 'CentOS',
'orchestrator_data': {
'puppet_manifests_source': 'path/to/manifests',
'puppet_modules_source': 'path/to/modules',
'repo_metadata': {
'repo': 'path/to/repo', }}}
self.validator = ReleaseValidator
def get_release(self, release):
return jsonutils.dumps(release)
def test_name_is_mandatory(self):
self.release.pop('name')
self.assertRaisesRegexp(
errors.InvalidData,
'No release name specified',
self.validator.validate,
self.get_release(self.release))
def test_version_is_mandatory(self):
self.release.pop('version')
self.assertRaisesRegexp(
errors.InvalidData,
'No release version specified',
self.validator.validate,
self.get_release(self.release))
def test_operating_system_is_mandatory(self):
self.release.pop('operating_system')
self.assertRaisesRegexp(
errors.InvalidData,
'No release operating system specified',
self.validator.validate,
self.get_release(self.release))
def test_orchestrator_data_is_mandatory(self):
self.release.pop('orchestrator_data')
self.assertRaisesRegexp(
errors.InvalidData,
'No orchestrator_data specified',
self.validator.validate,
self.get_release(self.release))
def test_orchestrator_data_must_be_a_dict(self):
self.release['orchestrator_data'] = None
self.assertRaisesRegexp(
errors.InvalidData,
"'orchestrator_data' field must be a dict",
self.validator.validate,
self.get_release(self.release))
def test_orchestrator_data_required_keys(self):
self.release['orchestrator_data'] = {}
self.assertRaisesRegexp(
errors.InvalidData,
"'orchestrator_data' doesn't have all required keys",
self.validator.validate,
self.get_release(self.release))
def test_default_are_good(self):
self.validator.validate(self.get_release(self.release))