Allow to tune image spec for every image
This requires changes in all component repos to replace FROM line to FROM {{ image_spec('someimage') }} Change-Id: I18281bdb41e91cd5c9160055f1617d7ee9d3b548
This commit is contained in:
parent
f48ba99b79
commit
2742f4bfc2
|
@ -13,6 +13,7 @@ import git
|
||||||
|
|
||||||
from fuel_ccp.common import jinja_utils
|
from fuel_ccp.common import jinja_utils
|
||||||
from fuel_ccp import config
|
from fuel_ccp import config
|
||||||
|
from fuel_ccp.config import images
|
||||||
|
|
||||||
BUILD_TIMEOUT = 2 ** 16 # in seconds
|
BUILD_TIMEOUT = 2 ** 16 # in seconds
|
||||||
|
|
||||||
|
@ -26,6 +27,7 @@ _SHUTDOWN = False
|
||||||
def render_dockerfile(path, name, config):
|
def render_dockerfile(path, name, config):
|
||||||
LOG.info('%s: Rendering dockerfile', name)
|
LOG.info('%s: Rendering dockerfile', name)
|
||||||
sources = set()
|
sources = set()
|
||||||
|
parent = [] # Could've been None if we could use nonlocal
|
||||||
|
|
||||||
def copy_sources(source_name, cont_dir):
|
def copy_sources(source_name, cont_dir):
|
||||||
if source_name not in config['sources']:
|
if source_name not in config['sources']:
|
||||||
|
@ -33,9 +35,16 @@ def render_dockerfile(path, name, config):
|
||||||
sources.add(source_name)
|
sources.add(source_name)
|
||||||
return 'COPY %s %s' % (source_name, cont_dir)
|
return 'COPY %s %s' % (source_name, cont_dir)
|
||||||
|
|
||||||
content = jinja_utils.jinja_render(path, config['render'], [copy_sources])
|
def image_spec(image_name):
|
||||||
|
if parent:
|
||||||
|
raise RuntimeError('You can use image_spec only once in FROM line')
|
||||||
|
parent.append(image_name)
|
||||||
|
return images.image_spec(image_name, add_address=CONF.builder.push)
|
||||||
|
|
||||||
return content, sources
|
content = jinja_utils.jinja_render(path, config['render'],
|
||||||
|
[copy_sources, image_spec])
|
||||||
|
|
||||||
|
return content, sources, parent[0] if parent else None
|
||||||
|
|
||||||
|
|
||||||
def prepare_source(source_name, name, dest_dir, config):
|
def prepare_source(source_name, name, dest_dir, config):
|
||||||
|
@ -84,19 +93,16 @@ def find_dockerfiles(repository_name, match=True):
|
||||||
dockerfiles = {}
|
dockerfiles = {}
|
||||||
repository_dir = os.path.join(CONF.repositories.path, repository_name)
|
repository_dir = os.path.join(CONF.repositories.path, repository_name)
|
||||||
|
|
||||||
namespace = CONF.images.namespace
|
|
||||||
if CONF.builder.push and CONF.registry.address:
|
|
||||||
namespace = '%s/%s' % (CONF.registry.address, namespace)
|
|
||||||
|
|
||||||
for root, __, files in os.walk(repository_dir):
|
for root, __, files in os.walk(repository_dir):
|
||||||
if 'Dockerfile.j2' in files:
|
if 'Dockerfile.j2' in files:
|
||||||
path = os.path.join(root, 'Dockerfile.j2')
|
path = os.path.join(root, 'Dockerfile.j2')
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
name = os.path.basename(os.path.dirname(path))
|
name = os.path.basename(os.path.dirname(path))
|
||||||
|
spec = images.image_spec(name, add_address=CONF.builder.push)
|
||||||
dockerfiles[name] = {
|
dockerfiles[name] = {
|
||||||
'name': name,
|
'name': name,
|
||||||
'full_name': '%s/%s:%s' % (namespace, name, CONF.images.tag),
|
'full_name': spec,
|
||||||
'path': path,
|
'path': path,
|
||||||
'parent': None,
|
'parent': None,
|
||||||
'children': [],
|
'children': [],
|
||||||
|
@ -120,44 +126,25 @@ def find_dockerfiles(repository_name, match=True):
|
||||||
|
|
||||||
def render_dockerfiles(dockerfiles, config):
|
def render_dockerfiles(dockerfiles, config):
|
||||||
for dockerfile in dockerfiles.values():
|
for dockerfile in dockerfiles.values():
|
||||||
content, sources = \
|
content, sources, parent = \
|
||||||
render_dockerfile(dockerfile['path'], dockerfile['name'], config)
|
render_dockerfile(dockerfile['path'], dockerfile['name'], config)
|
||||||
dockerfile['content'] = content
|
dockerfile['content'] = content
|
||||||
dockerfile['sources'] = sources
|
dockerfile['sources'] = sources
|
||||||
|
dockerfile['parent'] = parent
|
||||||
|
|
||||||
|
|
||||||
IMAGE_FULL_NAME_RE = r"((?P<namespace>[\w:\.-]+)/){0,2}" \
|
IMAGE_FULL_NAME_RE = r"((?P<namespace>[\w:\.-]+)/){0,2}" \
|
||||||
"(?P<name>[\w_-]+)" \
|
"(?P<name>[\w_-]+)" \
|
||||||
"(:(?P<tag>[\w_\.-]+))?"
|
"(:(?P<tag>[\w_\.-]+))?"
|
||||||
IMAGE_FULL_NAME_PATTERN = re.compile(IMAGE_FULL_NAME_RE)
|
IMAGE_FULL_NAME_PATTERN = re.compile(IMAGE_FULL_NAME_RE)
|
||||||
# This regex is needed for matching not yet rendered images
|
|
||||||
NOT_RENDERED_IMAGE_PATTERN = (r"((?P<namespace>[\w:\.\-}{ ]+)/){0,2}"
|
|
||||||
r"(?P<name>[\w_\-}{ ]+)"
|
|
||||||
r"(:(?P<tag>[\w_\.\-}{ ]+))?")
|
|
||||||
|
|
||||||
DOCKER_FILE_FROM_PATTERN = re.compile(
|
|
||||||
r"^\s?FROM\s+{}\s?$".format(NOT_RENDERED_IMAGE_PATTERN), re.MULTILINE
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def find_dependencies(dockerfiles):
|
def connect_children(dockerfiles):
|
||||||
for name, dockerfile in dockerfiles.items():
|
for dockerfile in dockerfiles.values():
|
||||||
with open(dockerfile['path']) as f:
|
parent = dockerfile['parent']
|
||||||
content = f.read()
|
if parent:
|
||||||
|
dockerfiles[parent]['children'].append(dockerfile)
|
||||||
matcher = DOCKER_FILE_FROM_PATTERN.search(content)
|
dockerfile['parent'] = dockerfiles[parent]
|
||||||
if not matcher:
|
|
||||||
raise RuntimeError(
|
|
||||||
"FROM clause was not found in dockerfile for image: " + name
|
|
||||||
)
|
|
||||||
|
|
||||||
parent_ns = matcher.group("namespace")
|
|
||||||
if not parent_ns:
|
|
||||||
continue
|
|
||||||
parent_name = matcher.group("name")
|
|
||||||
|
|
||||||
dockerfile['parent'] = dockerfiles[parent_name]
|
|
||||||
dockerfiles[parent_name]['children'].append(dockerfile)
|
|
||||||
|
|
||||||
|
|
||||||
def build_dockerfile(dc, dockerfile):
|
def build_dockerfile(dc, dockerfile):
|
||||||
|
@ -381,7 +368,7 @@ def build_components(components=None):
|
||||||
find_dockerfiles(repository_def['name'], match=match))
|
find_dockerfiles(repository_def['name'], match=match))
|
||||||
|
|
||||||
render_dockerfiles(dockerfiles, config)
|
render_dockerfiles(dockerfiles, config)
|
||||||
find_dependencies(dockerfiles)
|
connect_children(dockerfiles)
|
||||||
|
|
||||||
ready_images = get_ready_image_names()
|
ready_images = get_ready_image_names()
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ DEFAULTS = {
|
||||||
'base_distro': 'debian',
|
'base_distro': 'debian',
|
||||||
'base_tag': 'jessie',
|
'base_tag': 'jessie',
|
||||||
'maintainer': 'MOS Microservices <mos-microservices@mirantis.com>',
|
'maintainer': 'MOS Microservices <mos-microservices@mirantis.com>',
|
||||||
|
'image_specs': {},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -18,6 +19,34 @@ SCHEMA = {
|
||||||
'base_distro': {'type': 'string'},
|
'base_distro': {'type': 'string'},
|
||||||
'base_tag': {'type': 'string'},
|
'base_tag': {'type': 'string'},
|
||||||
'maintainer': {'type': 'string'},
|
'maintainer': {'type': 'string'},
|
||||||
|
'image_specs': {
|
||||||
|
'type': 'object',
|
||||||
|
'additionalProperties': {
|
||||||
|
'type': 'object',
|
||||||
|
'additionalProperties': False,
|
||||||
|
'properties': {
|
||||||
|
'tag': {'type': 'string'},
|
||||||
|
'namespace': {'type': 'string'},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def image_spec(image_name, add_address=True):
|
||||||
|
from fuel_ccp import config
|
||||||
|
CONF = config._REAL_CONF
|
||||||
|
spec = {
|
||||||
|
'name': image_name,
|
||||||
|
'namespace': CONF.images.namespace,
|
||||||
|
'tag': CONF.images.tag,
|
||||||
|
}
|
||||||
|
image_spec = CONF.images.image_specs.get(image_name)
|
||||||
|
if image_spec:
|
||||||
|
spec.update(image_spec._items())
|
||||||
|
spec_str = '{namespace}/{name}:{tag}'.format(**spec)
|
||||||
|
if add_address and CONF.registry.address:
|
||||||
|
spec_str = '{}/{}'.format(CONF.registry.address, spec_str)
|
||||||
|
return spec_str
|
||||||
|
|
|
@ -2,6 +2,7 @@ import json
|
||||||
|
|
||||||
from fuel_ccp.common import utils
|
from fuel_ccp.common import utils
|
||||||
from fuel_ccp import config
|
from fuel_ccp import config
|
||||||
|
from fuel_ccp.config import images
|
||||||
|
|
||||||
|
|
||||||
CONF = config.CONF
|
CONF = config.CONF
|
||||||
|
@ -16,14 +17,6 @@ ENTRYPOINT_PATH = "/opt/ccp_start_script/bin/start_script.py"
|
||||||
PYTHON_PATH = "/usr/bin/python"
|
PYTHON_PATH = "/usr/bin/python"
|
||||||
|
|
||||||
|
|
||||||
def _get_image_name(image_name):
|
|
||||||
image_name = "%s/%s:%s" % (CONF.images.namespace, image_name,
|
|
||||||
CONF.images.tag)
|
|
||||||
if CONF.registry.address:
|
|
||||||
image_name = "%s/%s" % (CONF.registry.address, image_name)
|
|
||||||
return image_name
|
|
||||||
|
|
||||||
|
|
||||||
def _get_start_cmd(role_name):
|
def _get_start_cmd(role_name):
|
||||||
return ["dumb-init", PYTHON_PATH, ENTRYPOINT_PATH, "provision", role_name]
|
return ["dumb-init", PYTHON_PATH, ENTRYPOINT_PATH, "provision", role_name]
|
||||||
|
|
||||||
|
@ -105,7 +98,7 @@ def serialize_env_variables(container):
|
||||||
def serialize_daemon_container_spec(container):
|
def serialize_daemon_container_spec(container):
|
||||||
cont_spec = {
|
cont_spec = {
|
||||||
"name": container["name"],
|
"name": container["name"],
|
||||||
"image": _get_image_name(container["image"]),
|
"image": images.image_spec(container["image"]),
|
||||||
"command": _get_start_cmd(container["name"]),
|
"command": _get_start_cmd(container["name"]),
|
||||||
"volumeMounts": serialize_volume_mounts(container),
|
"volumeMounts": serialize_volume_mounts(container),
|
||||||
"readinessProbe": {
|
"readinessProbe": {
|
||||||
|
@ -136,7 +129,7 @@ def serialize_daemon_container_spec(container):
|
||||||
def serialize_job_container_spec(container, job):
|
def serialize_job_container_spec(container, job):
|
||||||
return {
|
return {
|
||||||
"name": job["name"],
|
"name": job["name"],
|
||||||
"image": _get_image_name(container["image"]),
|
"image": images.image_spec(container["image"]),
|
||||||
"command": _get_start_cmd(job["name"]),
|
"command": _get_start_cmd(job["name"]),
|
||||||
"volumeMounts": serialize_volume_mounts(container),
|
"volumeMounts": serialize_volume_mounts(container),
|
||||||
"env": serialize_env_variables(container)
|
"env": serialize_env_variables(container)
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
import collections
|
import collections
|
||||||
import io
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import fixtures
|
import fixtures
|
||||||
|
@ -52,36 +51,6 @@ class TestBuild(base.TestCase):
|
||||||
},)
|
},)
|
||||||
])
|
])
|
||||||
|
|
||||||
def test_find_dependencies_no_registry(self):
|
|
||||||
m_open = mock.mock_open()
|
|
||||||
m_open.side_effect = [
|
|
||||||
io.StringIO(BASE_DOCKERFILE),
|
|
||||||
io.StringIO(COMPONENT_DOCKERFILE.format(''))
|
|
||||||
]
|
|
||||||
dockerfiles = self.__create_dockerfile_objects()
|
|
||||||
with mock.patch('fuel_ccp.build.open', m_open, create=True):
|
|
||||||
build.find_dependencies(dockerfiles)
|
|
||||||
|
|
||||||
self.assertListEqual([dockerfiles['ms-mysql']],
|
|
||||||
dockerfiles['ms-debian-base']['children'])
|
|
||||||
self.assertDictEqual(dockerfiles['ms-debian-base'],
|
|
||||||
dockerfiles['ms-mysql']['parent'])
|
|
||||||
|
|
||||||
def test_find_dependencies_registry(self):
|
|
||||||
m_open = mock.mock_open()
|
|
||||||
m_open.side_effect = [
|
|
||||||
io.StringIO(BASE_DOCKERFILE),
|
|
||||||
io.StringIO(COMPONENT_DOCKERFILE.format('example.com:8909/'))
|
|
||||||
]
|
|
||||||
dockerfiles = self.__create_dockerfile_objects()
|
|
||||||
with mock.patch('fuel_ccp.build.open', m_open, create=True):
|
|
||||||
build.find_dependencies(dockerfiles)
|
|
||||||
|
|
||||||
self.assertListEqual([dockerfiles['ms-mysql']],
|
|
||||||
dockerfiles['ms-debian-base']['children'])
|
|
||||||
self.assertDictEqual(dockerfiles['ms-debian-base'],
|
|
||||||
dockerfiles['ms-mysql']['parent'])
|
|
||||||
|
|
||||||
@mock.patch("docker.Client")
|
@mock.patch("docker.Client")
|
||||||
@mock.patch("fuel_ccp.build.build_dockerfile")
|
@mock.patch("fuel_ccp.build.build_dockerfile")
|
||||||
@mock.patch("fuel_ccp.build.submit_dockerfile_processing")
|
@mock.patch("fuel_ccp.build.submit_dockerfile_processing")
|
||||||
|
@ -266,18 +235,23 @@ class TestRenderDockerfile(testscenarios.WithScenarios, base.TestCase):
|
||||||
('empty', {
|
('empty', {
|
||||||
'config': {'render': {}},
|
'config': {'render': {}},
|
||||||
'source': '',
|
'source': '',
|
||||||
'result': ('', set()),
|
'result': ('', set(), None),
|
||||||
}),
|
}),
|
||||||
('one_source', {
|
('one_source', {
|
||||||
'config': {'render': {}, 'sources': {'one': {}}},
|
'config': {'render': {}, 'sources': {'one': {}}},
|
||||||
'source': '{{ copy_sources("one", "/tmp") }}',
|
'source': '{{ copy_sources("one", "/tmp") }}',
|
||||||
'result': ('COPY one /tmp', {'one'}),
|
'result': ('COPY one /tmp', {'one'}, None),
|
||||||
}),
|
}),
|
||||||
('wrong_source', {
|
('wrong_source', {
|
||||||
'config': {'render': {}, 'sources': {'one': {}}},
|
'config': {'render': {}, 'sources': {'one': {}}},
|
||||||
'source': '{{ copy_sources("wrong", "/tmp") }}',
|
'source': '{{ copy_sources("wrong", "/tmp") }}',
|
||||||
'exception': Exception('No such source: wrong'),
|
'exception': Exception('No such source: wrong'),
|
||||||
}),
|
}),
|
||||||
|
('one_from', {
|
||||||
|
'config': {'render': {}},
|
||||||
|
'source': 'FROM {{ image_spec("one") }}',
|
||||||
|
'result': ('FROM ccp/one:latest', set(), 'one'),
|
||||||
|
}),
|
||||||
]
|
]
|
||||||
|
|
||||||
config = None
|
config = None
|
||||||
|
|
Loading…
Reference in New Issue