Render only matched dockerfiles
Currently we are rendering all existing dockerfiles in repositories, but only requested for build should be rendered. Change-Id: Ib811ca24372fe42776e6458d2255bb82086dd70d Closes-bug: #1615981
This commit is contained in:
parent
e248c44ee2
commit
bab0c9b879
@ -29,6 +29,7 @@ _SHUTDOWN = False
|
||||
|
||||
|
||||
def create_rendered_dockerfile(path, name, tmp_path, config):
|
||||
LOG.info('%s: Rendering dockerfile', name)
|
||||
content = jinja_utils.jinja_render(path, config)
|
||||
src_dir = os.path.dirname(path)
|
||||
dest_dir = os.path.join(tmp_path, name)
|
||||
@ -49,7 +50,7 @@ def create_rendered_dockerfile(path, name, tmp_path, config):
|
||||
return dockerfilename
|
||||
|
||||
|
||||
def find_dockerfiles(repository_name, tmp_dir, config, match=True):
|
||||
def find_dockerfiles(repository_name, match=True):
|
||||
dockerfiles = {}
|
||||
repository_dir = os.path.join(CONF.repositories.path, repository_name)
|
||||
|
||||
@ -60,15 +61,9 @@ def find_dockerfiles(repository_name, tmp_dir, config, match=True):
|
||||
for root, __, files in os.walk(repository_dir):
|
||||
if 'Dockerfile.j2' in files:
|
||||
path = os.path.join(root, 'Dockerfile.j2')
|
||||
is_jinja2 = True
|
||||
elif 'Dockerfile' in files:
|
||||
path = os.path.join(root, 'Dockerfile')
|
||||
is_jinja2 = False
|
||||
else:
|
||||
continue
|
||||
name = os.path.basename(os.path.dirname(path))
|
||||
if is_jinja2:
|
||||
path = create_rendered_dockerfile(path, name, tmp_dir, config)
|
||||
dockerfiles[name] = {
|
||||
'name': name,
|
||||
'full_name': '%s/%s:%s' % (namespace, name, CONF.images.tag),
|
||||
@ -95,8 +90,13 @@ IMAGE_FULL_NAME_RE = r"((?P<namespace>[\w:\.-]+)/){0,2}" \
|
||||
"(?P<name>[\w_-]+)" \
|
||||
"(:(?P<tag>[\w_\.-]+))?"
|
||||
IMAGE_FULL_NAME_PATTERN = re.compile(IMAGE_FULL_NAME_RE)
|
||||
# This regex is needed for matching not yet rendered images
|
||||
NOT_RENDERED_IMAGE_PATTERN = (r"((?P<namespace>[\w:\.\-}{ ]+)/){0,2}"
|
||||
r"(?P<name>[\w_\-}{ ]+)"
|
||||
r"(:(?P<tag>[\w_\.\-}{ ]+))?")
|
||||
|
||||
DOCKER_FILE_FROM_PATTERN = re.compile(
|
||||
r"^\s?FROM\s+{}\s?$".format(IMAGE_FULL_NAME_RE), re.MULTILINE
|
||||
r"^\s?FROM\s+{}\s?$".format(NOT_RENDERED_IMAGE_PATTERN), re.MULTILINE
|
||||
)
|
||||
|
||||
|
||||
@ -112,10 +112,9 @@ def find_dependencies(dockerfiles):
|
||||
)
|
||||
|
||||
parent_ns = matcher.group("namespace")
|
||||
parent_name = matcher.group("name")
|
||||
|
||||
if CONF.images.namespace != parent_ns:
|
||||
if not parent_ns:
|
||||
continue
|
||||
parent_name = matcher.group("name")
|
||||
|
||||
dockerfile['parent'] = dockerfiles[parent_name]
|
||||
dockerfiles[parent_name]['children'].append(dockerfile)
|
||||
@ -184,7 +183,11 @@ def push_dockerfile(dc, dockerfile):
|
||||
CONF.registry.address)
|
||||
|
||||
|
||||
def process_dockerfile(dockerfile, executor, future_list, ready_images):
|
||||
def process_dockerfile(dockerfile, tmp_dir, config, executor, future_list,
|
||||
ready_images):
|
||||
path = create_rendered_dockerfile(
|
||||
dockerfile['path'], dockerfile['name'], tmp_dir, config)
|
||||
dockerfile['path'] = path
|
||||
with contextlib.closing(docker.Client(
|
||||
timeout=CONF.registry.timeout)) as dc:
|
||||
build_dockerfile(dc, dockerfile)
|
||||
@ -194,14 +197,15 @@ def process_dockerfile(dockerfile, executor, future_list, ready_images):
|
||||
for child in dockerfile['children']:
|
||||
if child['match'] or (CONF.builder.keep_image_tree_consistency and
|
||||
child['name'] in ready_images):
|
||||
submit_dockerfile_processing(child, executor, future_list,
|
||||
ready_images)
|
||||
submit_dockerfile_processing(child, tmp_dir, config, executor,
|
||||
future_list, ready_images)
|
||||
|
||||
|
||||
def submit_dockerfile_processing(dockerfile, executor, future_list,
|
||||
ready_images):
|
||||
def submit_dockerfile_processing(dockerfile, tmp_dir, config, executor,
|
||||
future_list, ready_images):
|
||||
future_list.append(executor.submit(
|
||||
process_dockerfile, dockerfile, executor, future_list, ready_images
|
||||
process_dockerfile, dockerfile, tmp_dir, config,
|
||||
executor, future_list, ready_images
|
||||
))
|
||||
|
||||
|
||||
@ -324,8 +328,7 @@ def build_components(components=None):
|
||||
match = not bool(components)
|
||||
for repository_name in CONF.repositories.names:
|
||||
dockerfiles.update(
|
||||
find_dockerfiles(
|
||||
repository_name, tmp_dir, config, match=match))
|
||||
find_dockerfiles(repository_name, match=match))
|
||||
|
||||
find_dependencies(dockerfiles)
|
||||
|
||||
@ -344,8 +347,9 @@ def build_components(components=None):
|
||||
if dockerfile['match'] and (
|
||||
dockerfile['parent'] is None or
|
||||
not dockerfile['parent']['match']):
|
||||
submit_dockerfile_processing(dockerfile, executor,
|
||||
future_list, ready_images)
|
||||
submit_dockerfile_processing(
|
||||
dockerfile, tmp_dir, config, executor,
|
||||
future_list, ready_images)
|
||||
|
||||
wait_futures(future_list)
|
||||
except SystemExit:
|
||||
|
@ -92,29 +92,34 @@ class TestBuild(base.TestCase):
|
||||
@mock.patch("docker.Client")
|
||||
@mock.patch("fuel_ccp.build.build_dockerfile")
|
||||
@mock.patch("fuel_ccp.build.submit_dockerfile_processing")
|
||||
def test_process_dockerfile_middle(self, submit_dockerfile_processing_mock,
|
||||
build_dockerfile_mock, dc_mock):
|
||||
@mock.patch("fuel_ccp.build.create_rendered_dockerfile")
|
||||
def test_process_dockerfile_middle(
|
||||
self, render_mock, submit_dockerfile_processing_mock,
|
||||
build_dockerfile_mock, dc_mock):
|
||||
dockerfiles = {
|
||||
'root': {
|
||||
'name': 'root',
|
||||
'full_name': 'ms/root',
|
||||
'parent': None,
|
||||
'children': ['middle'],
|
||||
'match': False
|
||||
'match': False,
|
||||
'path': '/tmp'
|
||||
},
|
||||
'middle': {
|
||||
'name': 'middle',
|
||||
'full_name': 'ms/middle',
|
||||
'parent': 'root',
|
||||
'children': ['leaf'],
|
||||
'match': True
|
||||
'match': True,
|
||||
'path': '/tmp'
|
||||
},
|
||||
'leaf': {
|
||||
'name': 'leaf',
|
||||
'full_name': 'ms/leaf',
|
||||
'parent': 'middle',
|
||||
'children': [],
|
||||
'match': False
|
||||
'match': False,
|
||||
'path': '/tmp'
|
||||
}
|
||||
}
|
||||
|
||||
@ -126,17 +131,20 @@ class TestBuild(base.TestCase):
|
||||
dockerfiles[dockerfile['children'][i]]
|
||||
)
|
||||
|
||||
build.process_dockerfile(dockerfiles["middle"], mock.ANY, mock.ANY,
|
||||
["root", "middle", "leaf"])
|
||||
build.process_dockerfile(
|
||||
dockerfiles["middle"], mock.ANY, mock.ANY, mock.ANY, mock.ANY,
|
||||
["root", "middle", "leaf"])
|
||||
|
||||
submit_dockerfile_processing_mock.assert_called_once_with(
|
||||
dockerfiles["leaf"], mock.ANY, mock.ANY, mock.ANY)
|
||||
dockerfiles["leaf"], mock.ANY, mock.ANY, mock.ANY,
|
||||
mock.ANY, ["root", "middle", "leaf"])
|
||||
|
||||
@mock.patch("docker.Client")
|
||||
@mock.patch("fuel_ccp.build.build_dockerfile")
|
||||
@mock.patch("fuel_ccp.build.submit_dockerfile_processing")
|
||||
@mock.patch("fuel_ccp.build.create_rendered_dockerfile")
|
||||
def test_process_dockerfile_middle_keep_consistency_off(
|
||||
self, submit_dockerfile_processing_mock,
|
||||
self, render_mock, submit_dockerfile_processing_mock,
|
||||
build_dockerfile_mock, dc_mock):
|
||||
dockerfiles = {
|
||||
'root': {
|
||||
@ -144,21 +152,24 @@ class TestBuild(base.TestCase):
|
||||
'full_name': 'ms/root',
|
||||
'parent': None,
|
||||
'children': ['middle'],
|
||||
'match': False
|
||||
'match': False,
|
||||
'path': '/tmp'
|
||||
},
|
||||
'middle': {
|
||||
'name': 'middle',
|
||||
'full_name': 'ms/middle',
|
||||
'parent': 'root',
|
||||
'children': ['leaf'],
|
||||
'match': True
|
||||
'match': True,
|
||||
'path': '/tmp'
|
||||
},
|
||||
'leaf': {
|
||||
'name': 'leaf',
|
||||
'full_name': 'ms/leaf',
|
||||
'parent': 'middle',
|
||||
'children': [],
|
||||
'match': False
|
||||
'match': False,
|
||||
'path': '/tmp'
|
||||
}
|
||||
}
|
||||
|
||||
@ -172,7 +183,8 @@ class TestBuild(base.TestCase):
|
||||
dockerfiles[dockerfile['children'][i]]
|
||||
)
|
||||
|
||||
build.process_dockerfile(dockerfiles["middle"], mock.ANY, mock.ANY, [])
|
||||
build.process_dockerfile(dockerfiles["middle"], mock.ANY, mock.ANY,
|
||||
mock.ANY, mock.ANY, [])
|
||||
|
||||
self.assertTrue(not submit_dockerfile_processing_mock.called)
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user