Add primitive support for specifying replicas
It's a minimal support for specifying number of replicas for services in the ccp config file. It's badly needed to simplify testing on scale and testing of the running services in HA. It's not breaking any kind of backward compatibility, you're now able to specify it: replicas: keystone: 3 nova-api: 42 Corresponding verifications was added into the process of building topology as it's implicitly part of it. Additionally, there is a very small tests refactoring plus few new tests for replicas verification. Change-Id: Ia7e032fd3d2090cf42b05dfb96d0f0808d1399d6
This commit is contained in:
parent
8c15db503d
commit
b552b0f91e
@ -9,6 +9,7 @@ from fuel_ccp.config import cli
|
|||||||
from fuel_ccp.config import images
|
from fuel_ccp.config import images
|
||||||
from fuel_ccp.config import kubernetes
|
from fuel_ccp.config import kubernetes
|
||||||
from fuel_ccp.config import registry
|
from fuel_ccp.config import registry
|
||||||
|
from fuel_ccp.config import replicas
|
||||||
from fuel_ccp.config import repositories
|
from fuel_ccp.config import repositories
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
@ -57,7 +58,8 @@ def get_config_defaults():
|
|||||||
'verbose_level': 1,
|
'verbose_level': 1,
|
||||||
'log_file': None,
|
'log_file': None,
|
||||||
})
|
})
|
||||||
for module in [cli, builder, images, kubernetes, registry, repositories]:
|
for module in [cli, builder, images, kubernetes, registry, replicas,
|
||||||
|
repositories]:
|
||||||
defaults._merge(module.DEFAULTS)
|
defaults._merge(module.DEFAULTS)
|
||||||
return defaults
|
return defaults
|
||||||
|
|
||||||
@ -72,7 +74,8 @@ def get_config_schema():
|
|||||||
'log_file': {'anyOf': [{'type': 'null'}, {'type': 'string'}]},
|
'log_file': {'anyOf': [{'type': 'null'}, {'type': 'string'}]},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for module in [cli, builder, images, kubernetes, registry, repositories]:
|
for module in [cli, builder, images, kubernetes, registry, replicas,
|
||||||
|
repositories]:
|
||||||
schema['properties'].update(module.SCHEMA)
|
schema['properties'].update(module.SCHEMA)
|
||||||
# Don't validate all options used to be added from oslo.log and oslo.config
|
# Don't validate all options used to be added from oslo.log and oslo.config
|
||||||
ignore_opts = ['debug', 'verbose', 'log_file']
|
ignore_opts = ['debug', 'verbose', 'log_file']
|
||||||
|
12
fuel_ccp/config/replicas.py
Normal file
12
fuel_ccp/config/replicas.py
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
DEFAULTS = {
|
||||||
|
}
|
||||||
|
|
||||||
|
SCHEMA = {
|
||||||
|
'replicas': {
|
||||||
|
'type': 'object',
|
||||||
|
"additionalProperties": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
@ -3,6 +3,7 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import six
|
||||||
|
|
||||||
from fuel_ccp.common import jinja_utils
|
from fuel_ccp.common import jinja_utils
|
||||||
from fuel_ccp.common import utils
|
from fuel_ccp.common import utils
|
||||||
@ -59,19 +60,21 @@ def _get_service_files_hash(service_dir, files, configs):
|
|||||||
|
|
||||||
def parse_role(service_dir, role, config):
|
def parse_role(service_dir, role, config):
|
||||||
service = role["service"]
|
service = role["service"]
|
||||||
if service["name"] not in config.get("topology", {}):
|
service_name = service["name"]
|
||||||
|
|
||||||
|
if service_name not in config.get("topology", {}):
|
||||||
LOG.info("Service %s not in topology config, skipping deploy",
|
LOG.info("Service %s not in topology config, skipping deploy",
|
||||||
service["name"])
|
service_name)
|
||||||
return
|
return
|
||||||
LOG.info("Scheduling service %s deployment", service["name"])
|
LOG.info("Scheduling service %s deployment", service_name)
|
||||||
_expand_files(service, role.get("files"))
|
_expand_files(service, role.get("files"))
|
||||||
|
|
||||||
files_cm = _create_files_configmap(
|
files_cm = _create_files_configmap(
|
||||||
service_dir, service["name"], role.get("files"))
|
service_dir, service_name, role.get("files"))
|
||||||
meta_cm = _create_meta_configmap(service)
|
meta_cm = _create_meta_configmap(service)
|
||||||
|
|
||||||
workflows = _parse_workflows(service)
|
workflows = _parse_workflows(service)
|
||||||
workflow_cm = _create_workflow(workflows, service["name"])
|
workflow_cm = _create_workflow(workflows, service_name)
|
||||||
configmaps = config['configmaps'] + (files_cm, meta_cm, workflow_cm)
|
configmaps = config['configmaps'] + (files_cm, meta_cm, workflow_cm)
|
||||||
|
|
||||||
if CONF.action.dry_run:
|
if CONF.action.dry_run:
|
||||||
@ -91,16 +94,26 @@ def parse_role(service_dir, role, config):
|
|||||||
cont_spec = templates.serialize_daemon_pod_spec(service)
|
cont_spec = templates.serialize_daemon_pod_spec(service)
|
||||||
affinity = templates.serialize_affinity(service, config["topology"])
|
affinity = templates.serialize_affinity(service, config["topology"])
|
||||||
|
|
||||||
|
replicas = config.get("replicas", {}).get(service_name)
|
||||||
if service.get("daemonset", False):
|
if service.get("daemonset", False):
|
||||||
obj = templates.serialize_daemonset(service["name"], cont_spec,
|
if replicas is not None:
|
||||||
|
LOG.error("Replicas was specified for %s, but it's implemented "
|
||||||
|
"using Kubernetes DaemonSet that will deploy service on "
|
||||||
|
"all matching nodes (section 'nodes' in config file)",
|
||||||
|
service_name)
|
||||||
|
raise RuntimeError("Replicas couldn't be specified for services "
|
||||||
|
"implemented using Kubernetes DaemonSet")
|
||||||
|
|
||||||
|
obj = templates.serialize_daemonset(service_name, cont_spec,
|
||||||
affinity)
|
affinity)
|
||||||
else:
|
else:
|
||||||
obj = templates.serialize_deployment(service["name"], cont_spec,
|
replicas = replicas or 1
|
||||||
affinity)
|
obj = templates.serialize_deployment(service_name, cont_spec,
|
||||||
|
affinity, replicas)
|
||||||
kubernetes.process_object(obj)
|
kubernetes.process_object(obj)
|
||||||
|
|
||||||
_create_service(service)
|
_create_service(service)
|
||||||
LOG.info("Service %s successfuly scheduled", service["name"])
|
LOG.info("Service %s successfuly scheduled", service_name)
|
||||||
|
|
||||||
|
|
||||||
def _parse_workflows(service):
|
def _parse_workflows(service):
|
||||||
@ -291,7 +304,7 @@ def _create_meta_configmap(service):
|
|||||||
return kubernetes.process_object(template)
|
return kubernetes.process_object(template)
|
||||||
|
|
||||||
|
|
||||||
def _make_topology(nodes, roles):
|
def _make_topology(nodes, roles, replicas):
|
||||||
failed = False
|
failed = False
|
||||||
# TODO(sreshetniak): move it to validation
|
# TODO(sreshetniak): move it to validation
|
||||||
if not nodes:
|
if not nodes:
|
||||||
@ -303,6 +316,9 @@ def _make_topology(nodes, roles):
|
|||||||
if failed:
|
if failed:
|
||||||
raise RuntimeError("Failed to create topology for services")
|
raise RuntimeError("Failed to create topology for services")
|
||||||
|
|
||||||
|
# Replicas are optional, 1 replica will deployed by default
|
||||||
|
replicas = replicas or dict()
|
||||||
|
|
||||||
# TODO(sreshetniak): add validation
|
# TODO(sreshetniak): add validation
|
||||||
k8s_nodes = kubernetes.list_k8s_nodes()
|
k8s_nodes = kubernetes.list_k8s_nodes()
|
||||||
k8s_node_names = kubernetes.get_object_names(k8s_nodes)
|
k8s_node_names = kubernetes.get_object_names(k8s_nodes)
|
||||||
@ -330,6 +346,26 @@ def _make_topology(nodes, roles):
|
|||||||
service_to_node[svc].extend(roles_to_node[role])
|
service_to_node[svc].extend(roles_to_node[role])
|
||||||
else:
|
else:
|
||||||
LOG.warning("Role '%s' defined, but unused", role)
|
LOG.warning("Role '%s' defined, but unused", role)
|
||||||
|
|
||||||
|
replicas = replicas.copy()
|
||||||
|
for svc, svc_hosts in six.iteritems(service_to_node):
|
||||||
|
svc_replicas = replicas.pop(svc, None)
|
||||||
|
|
||||||
|
if svc_replicas is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
svc_hosts_count = len(svc_hosts)
|
||||||
|
if svc_replicas > svc_hosts_count:
|
||||||
|
LOG.error("Requested %s replicas for %s while only %s hosts able "
|
||||||
|
"to run that service (%s)", svc_replicas, svc,
|
||||||
|
svc_hosts_count, ", ".join(svc_hosts))
|
||||||
|
raise RuntimeError("Replicas doesn't match available hosts.")
|
||||||
|
|
||||||
|
if replicas:
|
||||||
|
LOG.error("Replicas defined for unspecified service(s): %s",
|
||||||
|
", ".join(replicas.keys()))
|
||||||
|
raise RuntimeError("Replicas defined for unspecified service(s)")
|
||||||
|
|
||||||
return service_to_node
|
return service_to_node
|
||||||
|
|
||||||
|
|
||||||
@ -365,9 +401,11 @@ def deploy_components(components_map, components):
|
|||||||
if CONF.action.export_dir:
|
if CONF.action.export_dir:
|
||||||
os.makedirs(os.path.join(CONF.action.export_dir, 'configmaps'))
|
os.makedirs(os.path.join(CONF.action.export_dir, 'configmaps'))
|
||||||
|
|
||||||
config = utils.get_global_parameters("configs", "nodes", "roles")
|
config = utils.get_global_parameters("configs", "nodes", "roles",
|
||||||
|
"replicas")
|
||||||
config["topology"] = _make_topology(config.get("nodes"),
|
config["topology"] = _make_topology(config.get("nodes"),
|
||||||
config.get("roles"))
|
config.get("roles"),
|
||||||
|
config.get("replicas"))
|
||||||
|
|
||||||
namespace = CONF.kubernetes.namespace
|
namespace = CONF.kubernetes.namespace
|
||||||
_create_namespace(namespace)
|
_create_namespace(namespace)
|
||||||
|
@ -260,7 +260,7 @@ def serialize_job(name, spec):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def serialize_deployment(name, spec, affinity):
|
def serialize_deployment(name, spec, affinity, replicas):
|
||||||
return {
|
return {
|
||||||
"apiVersion": "extensions/v1beta1",
|
"apiVersion": "extensions/v1beta1",
|
||||||
"kind": "Deployment",
|
"kind": "Deployment",
|
||||||
@ -268,7 +268,7 @@ def serialize_deployment(name, spec, affinity):
|
|||||||
"name": name
|
"name": name
|
||||||
},
|
},
|
||||||
"spec": {
|
"spec": {
|
||||||
"replicas": 1,
|
"replicas": replicas,
|
||||||
"strategy": {
|
"strategy": {
|
||||||
"rollingUpdate": {
|
"rollingUpdate": {
|
||||||
"maxSurge": 1,
|
"maxSurge": 1,
|
||||||
|
@ -321,20 +321,11 @@ class TestDeployMakeTopology(base.TestCase):
|
|||||||
self.useFixture(
|
self.useFixture(
|
||||||
fixtures.MockPatch("fuel_ccp.kubernetes.list_k8s_nodes"))
|
fixtures.MockPatch("fuel_ccp.kubernetes.list_k8s_nodes"))
|
||||||
|
|
||||||
def test_make_empty_topology(self):
|
|
||||||
self.assertRaises(RuntimeError,
|
|
||||||
deploy._make_topology, None, None)
|
|
||||||
self.assertRaises(RuntimeError,
|
|
||||||
deploy._make_topology, None, {"spam": "eggs"})
|
|
||||||
self.assertRaises(RuntimeError,
|
|
||||||
deploy._make_topology, {"spam": "eggs"}, None)
|
|
||||||
|
|
||||||
def test_make_topology(self):
|
|
||||||
node_list = ["node1", "node2", "node3"]
|
node_list = ["node1", "node2", "node3"]
|
||||||
self.useFixture(fixtures.MockPatch(
|
self.useFixture(fixtures.MockPatch(
|
||||||
"fuel_ccp.kubernetes.get_object_names", return_value=node_list))
|
"fuel_ccp.kubernetes.get_object_names", return_value=node_list))
|
||||||
|
|
||||||
roles = {
|
self._roles = {
|
||||||
"controller": [
|
"controller": [
|
||||||
"mysql",
|
"mysql",
|
||||||
"keystone"
|
"keystone"
|
||||||
@ -345,6 +336,15 @@ class TestDeployMakeTopology(base.TestCase):
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def test_make_empty_topology(self):
|
||||||
|
self.assertRaises(RuntimeError,
|
||||||
|
deploy._make_topology, None, None, None)
|
||||||
|
self.assertRaises(RuntimeError,
|
||||||
|
deploy._make_topology, None, {"spam": "eggs"}, None)
|
||||||
|
self.assertRaises(RuntimeError,
|
||||||
|
deploy._make_topology, {"spam": "eggs"}, None, None)
|
||||||
|
|
||||||
|
def test_make_topology_without_replicas(self):
|
||||||
nodes = {
|
nodes = {
|
||||||
"node1": {
|
"node1": {
|
||||||
"roles": ["controller"]
|
"roles": ["controller"]
|
||||||
@ -361,10 +361,10 @@ class TestDeployMakeTopology(base.TestCase):
|
|||||||
"libvirtd": ["node2", "node3"]
|
"libvirtd": ["node2", "node3"]
|
||||||
}
|
}
|
||||||
|
|
||||||
topology = deploy._make_topology(nodes, roles)
|
topology = deploy._make_topology(nodes, self._roles, None)
|
||||||
self.assertDictEqual(expected_topology, topology)
|
self.assertDictEqual(expected_topology, topology)
|
||||||
|
|
||||||
# check if role is defined but not used
|
def test_make_topology_without_replicas_unused_role(self):
|
||||||
nodes = {
|
nodes = {
|
||||||
"node1": {
|
"node1": {
|
||||||
"roles": ["controller"]
|
"roles": ["controller"]
|
||||||
@ -375,11 +375,11 @@ class TestDeployMakeTopology(base.TestCase):
|
|||||||
"mysql": ["node1"],
|
"mysql": ["node1"],
|
||||||
"keystone": ["node1"]
|
"keystone": ["node1"]
|
||||||
}
|
}
|
||||||
topology = deploy._make_topology(nodes, roles)
|
|
||||||
|
topology = deploy._make_topology(nodes, self._roles, None)
|
||||||
self.assertDictEqual(expected_topology, topology)
|
self.assertDictEqual(expected_topology, topology)
|
||||||
|
|
||||||
# two ways to define topology that should give the same result
|
def test_make_topology_without_replicas_twice_used_role(self):
|
||||||
# first
|
|
||||||
nodes = {
|
nodes = {
|
||||||
"node1": {
|
"node1": {
|
||||||
"roles": ["controller", "compute"]
|
"roles": ["controller", "compute"]
|
||||||
@ -395,10 +395,10 @@ class TestDeployMakeTopology(base.TestCase):
|
|||||||
"nova-compute": ["node1", "node2", "node3"],
|
"nova-compute": ["node1", "node2", "node3"],
|
||||||
"libvirtd": ["node1", "node2", "node3"]
|
"libvirtd": ["node1", "node2", "node3"]
|
||||||
}
|
}
|
||||||
topology = deploy._make_topology(nodes, roles)
|
topology = deploy._make_topology(nodes, self._roles, None)
|
||||||
self.assertDictEqual(expected_topology, topology)
|
self.assertDictEqual(expected_topology, topology)
|
||||||
|
|
||||||
# second
|
def test_make_topology_without_replicas_twice_used_node(self):
|
||||||
nodes = {
|
nodes = {
|
||||||
"node1": {
|
"node1": {
|
||||||
"roles": ["controller"]
|
"roles": ["controller"]
|
||||||
@ -414,5 +414,30 @@ class TestDeployMakeTopology(base.TestCase):
|
|||||||
"nova-compute": ["node1", "node2", "node3"],
|
"nova-compute": ["node1", "node2", "node3"],
|
||||||
"libvirtd": ["node1", "node2", "node3"]
|
"libvirtd": ["node1", "node2", "node3"]
|
||||||
}
|
}
|
||||||
topology = deploy._make_topology(nodes, roles)
|
|
||||||
|
topology = deploy._make_topology(nodes, self._roles, None)
|
||||||
self.assertDictEqual(expected_topology, topology)
|
self.assertDictEqual(expected_topology, topology)
|
||||||
|
|
||||||
|
def test_make_topology_replicas_bigger_than_nodes(self):
|
||||||
|
replicas = {
|
||||||
|
"keystone": 2
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes = {
|
||||||
|
"node1": {
|
||||||
|
"roles": ["controller"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.assertRaises(RuntimeError,
|
||||||
|
deploy._make_topology, nodes, self._roles, replicas)
|
||||||
|
|
||||||
|
def test_make_topology_unspecified_service_replicas(self):
|
||||||
|
replicas = {
|
||||||
|
"foobar": 42
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes = {}
|
||||||
|
|
||||||
|
self.assertRaises(RuntimeError,
|
||||||
|
deploy._make_topology, nodes, self._roles, replicas)
|
||||||
|
Loading…
Reference in New Issue
Block a user