diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py index dee1bb7902..30c9dbd51a 100755 --- a/kolla/cmd/build.py +++ b/kolla/cmd/build.py @@ -80,6 +80,51 @@ class KollaRpmSetupUnknownConfig(Exception): pass +# Image status constants. +# +# TODO(harlowja): use enum lib in the future?? +STATUS_CONNECTION_ERROR = 'connection_error' +STATUS_PUSH_ERROR = 'push_error' +STATUS_ERROR = 'error' +STATUS_PARENT_ERROR = 'parent_error' +STATUS_BUILT = 'built' +STATUS_BUILDING = 'building' +STATUS_UNMATCHED = 'unmatched' +STATUS_MATCHED = 'matched' +STATUS_UNPROCESSED = 'unprocessed' + +# All error status constants. +STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR, + STATUS_ERROR, STATUS_PARENT_ERROR) + + +class Recorder(object): + """Recorder/buffer of (unicode) log lines for eventual display.""" + + def __init__(self): + self._lines = [] + + def write(self, text=""): + if isinstance(text, six.text_type): + self._lines.append(text) + elif isinstance(text, six.binary_type): + self._lines.append(text.decode('utf8')) + elif isinstance(text, Recorder): + self._lines.extend(text._lines) + else: + self.write(text=str(text)) + + def clear(self): + self._lines = [] + + def __iter__(self): + for line in self._lines: + yield line + + def __str__(self): + return u"\n".join(self._lines) + + def docker_client(): try: docker_kwargs = docker.utils.kwargs_from_env() @@ -90,6 +135,28 @@ def docker_client(): sys.exit(1) +class Image(object): + def __init__(self, name, canonical_name, path, parent_name='', + status=STATUS_UNPROCESSED, parent=None, source=None): + self.name = name + self.canonical_name = canonical_name + self.path = path + self.status = status + self.parent = parent + self.source = source + self.parent_name = parent_name + self.logs = Recorder() + self.push_logs = Recorder() + self.children = [] + self.plugins = [] + + def __repr__(self): + return ("Image(%s, %s, %s, parent_name=%s," + " status=%s, parent=%s, source=%s)") % ( + self.name, self.canonical_name, self.path, + self.parent_name, self.status, self.parent, self.source) + + class PushIntoQueueTask(task.Task): """Task that pushes some other task into a queue.""" @@ -119,41 +186,41 @@ class PushTask(task.Task): @property def name(self): - return 'PushTask(%s)' % self.image['name'] + return 'PushTask(%s)' % self.image.name def run(self): image = self.image try: - LOG.debug('%s:Try to push the image', image['name']) + LOG.debug('%s:Try to push the image', image.name) self.push_image(image) except requests_exc.ConnectionError: LOG.exception('%s:Make sure Docker is running and that you' ' have the correct privileges to run Docker' - ' (root)', image['name']) - image['status'] = "connection_error" + ' (root)', image.name) + image.status = STATUS_CONNECTION_ERROR except Exception: - LOG.exception('%s:Unknown error when pushing', image['name']) - image['status'] = "push_error" + LOG.exception('%s:Unknown error when pushing', image.name) + image.status = STATUS_PUSH_ERROR finally: - if "error" not in image['status']: - LOG.info('%s:Pushed successfully', image['name']) + if (image.status not in STATUS_ERRORS + and image.status != STATUS_UNPROCESSED): + LOG.info('%s:Pushed successfully', image.name) self.success = True else: self.success = False def push_image(self, image): - image['push_logs'] = str() - - for response in self.dc.push(image['fullname'], + image.push_logs.clear() + for response in self.dc.push(image.canonical_name, stream=True, insecure_registry=True): stream = json.loads(response) - if 'stream' in stream: - image['push_logs'] = image['logs'] + stream['stream'] + image.push_logs.write(image.logs) + image.push_logs.write(stream['stream']) LOG.info('%s', stream['stream']) elif 'errorDetail' in stream: - image['status'] = "error" + image.status = STATUS_ERROR LOG.error(stream['errorDetail']['message']) @@ -171,11 +238,11 @@ class BuildTask(task.Task): @property def name(self): - return 'BuildTask(%s)' % self.image['name'] + return 'BuildTask(%s)' % self.image.name def run(self): self.builder(self.image) - if self.image['status'] == 'built': + if self.image.status == STATUS_BUILT: self.success = True @property @@ -189,24 +256,24 @@ class BuildTask(task.Task): PushTask(self.conf, self.image), self.push_queue), ]) - if self.image['children'] and self.success: - for image in self.image['children']: + if self.image.children and self.success: + for image in self.image.children: followups.append(BuildTask(self.conf, image, self.push_queue)) return followups def process_source(self, image, source): - dest_archive = os.path.join(image['path'], source['name'] + '-archive') + dest_archive = os.path.join(image.path, source['name'] + '-archive') if source.get('type') == 'url': - LOG.debug("%s:Getting archive from %s", image['name'], + LOG.debug("%s:Getting archive from %s", image.name, source['source']) try: r = requests.get(source['source'], timeout=self.conf.timeout) except requests_exc.Timeout: LOG.exception('Request timed out while getting archive' ' from %s', source['source']) - image['status'] = "error" - image['logs'] = str() + image.status = STATUS_ERROR + image.logs.clear() return if r.status_code == 200: @@ -214,34 +281,34 @@ class BuildTask(task.Task): f.write(r.content) else: LOG.error('%s:Failed to download archive: status_code %s', - image['name'], r.status_code) - image['status'] = "error" + image.name, r.status_code) + image.status = STATUS_ERROR return elif source.get('type') == 'git': clone_dir = '{}-{}'.format(dest_archive, source['reference'].replace('/', '-')) try: - LOG.debug("%s:Cloning from %s", image['name'], + LOG.debug("%s:Cloning from %s", image.name, source['source']) git.Git().clone(source['source'], clone_dir) git.Git(clone_dir).checkout(source['reference']) reference_sha = git.Git(clone_dir).rev_parse('HEAD') LOG.debug("%s:Git checkout by reference %s (%s)", - image['name'], source['reference'], reference_sha) + image.name, source['reference'], reference_sha) except Exception as e: - LOG.error("%s:Failed to get source from git", image['name']) - LOG.error("%s:Error:%s", image['name'], str(e)) + LOG.error("%s:Failed to get source from git", image.name) + LOG.error("%s:Error:%s", image.name, str(e)) # clean-up clone folder to retry shutil.rmtree(clone_dir) - image['status'] = "error" + image.status = STATUS_ERROR return with tarfile.open(dest_archive, 'w') as tar: tar.add(clone_dir, arcname=os.path.basename(clone_dir)) elif source.get('type') == 'local': - LOG.debug("%s:Getting local archive from %s", image['name'], + LOG.debug("%s:Getting local archive from %s", image.name, source['source']) if os.path.isdir(source['source']): with tarfile.open(dest_archive, 'w') as tar: @@ -251,9 +318,9 @@ class BuildTask(task.Task): shutil.copyfile(source['source'], dest_archive) else: - LOG.error("%s:Wrong source type '%s'", image['name'], + LOG.error("%s:Wrong source type '%s'", image.name, source.get('type')) - image['status'] = "error" + image.status = STATUS_ERROR return # Set time on destination archive to epoch 0 @@ -279,31 +346,30 @@ class BuildTask(task.Task): return buildargs def builder(self, image): - LOG.debug('%s:Processing', image['name']) - if image['status'] == 'unmatched': + LOG.debug('%s:Processing', image.name) + if image.status == STATUS_UNMATCHED: return - if (image['parent'] is not None and - image['parent']['status'] in ['error', 'parent_error', - 'connection_error']): + if (image.parent is not None and + image.parent.status in STATUS_ERRORS): LOG.error('%s:Parent image error\'d with message "%s"', - image['name'], image['parent']['status']) - image['status'] = "parent_error" + image.name, image.parent.status) + image.status = STATUS_PARENT_ERROR return - image['status'] = "building" - LOG.info('%s:Building', image['name']) + image.status = STATUS_BUILDING + LOG.info('%s:Building', image.name) - if 'source' in image and 'source' in image['source']: - self.process_source(image, image['source']) - if image['status'] == "error": + if image.source and 'source' in image.source: + self.process_source(image, image.source) + if image.status in STATUS_ERRORS: return plugin_archives = list() - plugins_path = os.path.join(image['path'], 'plugins') - for plugin in image['plugins']: + plugins_path = os.path.join(image.path, 'plugins') + for plugin in image.plugins: archive_path = self.process_source(image, plugin) - if image['status'] == "error": + if image.status in STATUS_ERRORS: return plugin_archives.append(archive_path) if plugin_archives: @@ -320,19 +386,19 @@ class BuildTask(task.Task): else: LOG.error('Failed to create directory %s: %s', plugins_path, e) - image['status'] = "error" + image.status = STATUS_CONNECTION_ERROR return - with tarfile.open(os.path.join(image['path'], 'plugins-archive'), + with tarfile.open(os.path.join(image.path, 'plugins-archive'), 'w') as tar: tar.add(plugins_path, arcname='plugins') # Pull the latest image for the base distro only - pull = True if image['parent'] is None else False + pull = True if image.parent is None else False - image['logs'] = str() + image.logs.clear() buildargs = self.update_buildargs() - for response in self.dc.build(path=image['path'], - tag=image['fullname'], + for response in self.dc.build(path=image.path, + tag=image.canonical_name, nocache=self.nocache, rm=True, pull=pull, @@ -341,23 +407,23 @@ class BuildTask(task.Task): stream = json.loads(response.decode('utf-8')) if 'stream' in stream: - image['logs'] = image['logs'] + stream['stream'] + image.logs.write(stream['stream']) for line in stream['stream'].split('\n'): if line: - LOG.info('%s:%s', image['name'], line) + LOG.info('%s:%s', image.name, line) if 'errorDetail' in stream: - image['status'] = "error" + image.status = STATUS_ERROR LOG.error('%s:Error\'d with the following message', - image['name']) + image.name) for line in stream['errorDetail']['message'].split('\n'): if line: - LOG.error('%s:%s', image['name'], line) + LOG.error('%s:%s', image.name, line) return - image['status'] = "built" + image.status = STATUS_BUILT - LOG.info('%s:Built', image['name']) + LOG.info('%s:Built', image.name) class WorkerThread(threading.Thread): @@ -606,31 +672,31 @@ class KollaWorker(object): if filter_: patterns = re.compile(r"|".join(filter_).join('()')) for image in self.images: - if image['status'] == 'matched': + if image.status == STATUS_MATCHED: continue - if re.search(patterns, image['name']): - image['status'] = 'matched' - while (image['parent'] is not None and - image['parent']['status'] != 'matched'): - image = image['parent'] - image['status'] = 'matched' - LOG.debug('%s:Matched regex', image['name']) + if re.search(patterns, image.name): + image.status = STATUS_MATCHED + while (image.parent is not None and + image.parent.status != STATUS_MATCHED): + image = image.parent + image.status = STATUS_MATCHED + LOG.debug('%s:Matched regex', image.name) else: - image['status'] = 'unmatched' + image.status = STATUS_UNMATCHED else: for image in self.images: - image['status'] = 'matched' + image.status = STATUS_MATCHED def summary(self): """Walk the dictionary of images statuses and print results""" # For debug we print the logs again if the image error'd. This is to # to help us debug and it will be extra helpful in the gate. for image in self.images: - if image['status'] == 'error': - LOG.debug("%s:Failed with the following logs", image['name']) - for line in image['logs'].split('\n'): + if image.status in STATUS_ERRORS: + LOG.debug("%s:Failed with the following logs", image.name) + for line in image.logs: if line: - LOG.debug("%s:%s", image['name'], ''.join(line)) + LOG.debug("%s:%s", image.name, line) self.get_image_statuses() @@ -660,12 +726,12 @@ class KollaWorker(object): self.image_statuses_good, self.image_statuses_unmatched) for image in self.images: - if image['status'] == "built": - self.image_statuses_good[image['name']] = image['status'] - elif image['status'] == "unmatched": - self.image_statuses_unmatched[image['name']] = image['status'] + if image.status == STATUS_BUILT: + self.image_statuses_good[image.name] = image.status + elif image.status == STATUS_UNMATCHED: + self.image_statuses_unmatched[image.name] = image.status else: - self.image_statuses_bad[image['name']] = image['status'] + self.image_statuses_bad[image.name] = image.status return (self.image_statuses_bad, self.image_statuses_good, self.image_statuses_unmatched) @@ -689,33 +755,26 @@ class KollaWorker(object): with open(os.path.join(path, 'Dockerfile')) as f: content = f.read() - image = dict() - image['status'] = "unprocessed" - image['name'] = os.path.basename(path) - image['fullname'] = self.namespace + '/' + self.image_prefix + \ - image['name'] + ':' + self.tag - image['path'] = path - image['parent_name'] = content.split(' ')[1].split('\n')[0] - if not image['parent_name'].startswith(self.namespace + '/'): - image['parent'] = None - image['children'] = list() - image['plugins'] = list() + image_name = os.path.basename(path) + canonical_name = (self.namespace + '/' + self.image_prefix + + image_name + ':' + self.tag) + image = Image(image_name, canonical_name, path, + parent_name=content.split(' ')[1].split('\n')[0]) if self.install_type == 'source': # NOTE(jeffrey4l): register the opts if the section didn't # register in the kolla/common/config.py file - if image['name'] not in self.conf._groups: + if image.name not in self.conf._groups: self.conf.register_opts(common_config.get_source_opts(), - image['name']) - image['source'] = process_source_installation(image, - image['name']) + image.name) + image.source = process_source_installation(image, image.name) for plugin in [match.group(0) for match in - (re.search('{}-plugin-.+'.format(image['name']), + (re.search('{}-plugin-.+'.format(image.name), section) for section in self.conf.list_all_sections()) if match]: self.conf.register_opts(common_config.get_source_opts(), plugin) - image['plugins'].append( + image.plugins.append( process_source_installation(image, plugin)) self.images.append(image) @@ -724,25 +783,25 @@ class KollaWorker(object): dot = graphviz.Digraph(comment='Docker Images Dependency') dot.body.extend(['rankdir=LR']) for image in self.images: - if image['status'] not in ['matched']: + if image.status not in [STATUS_MATCHED]: continue - dot.node(image['name']) - if image['parent'] is not None: - dot.edge(image['parent']['name'], image['name']) + dot.node(image.name) + if image.parent is not None: + dot.edge(image.parent.name, image.name) with open(to_file, 'w') as f: f.write(dot.source) def list_images(self): for count, image in enumerate(self.images): - print(count + 1, ':', image['name']) + print(count + 1, ':', image.name) def list_dependencies(self): match = False for image in self.images: - if image['status'] in ['matched']: + if image.status in [STATUS_MATCHED]: match = True - if image['parent'] is None: + if image.parent is None: base = image if not match: print('Nothing matched!') @@ -751,18 +810,17 @@ class KollaWorker(object): def list_children(images, ancestry): children = ancestry.values()[0] for item in images: - if item['status'] not in ['matched']: + if item.status not in [STATUS_MATCHED]: continue - - if not item['children']: - children.append(item['name']) + if not item.children: + children.append(item.name) else: - newparent = {item['name']: []} + newparent = {item.name: []} children.append(newparent) - list_children(item['children'], newparent) + list_children(item.children, newparent) - ancestry = {base['name']: []} - list_children(base['children'], ancestry) + ancestry = {base.name: []} + list_children(base.children, ancestry) pprint.pprint(ancestry) def find_parents(self): @@ -770,13 +828,13 @@ class KollaWorker(object): sort_images = dict() for image in self.images: - sort_images[image['fullname']] = image + sort_images[image.canonical_name] = image for parent_name, parent in sort_images.items(): for image in sort_images.values(): - if image['parent_name'] == parent_name: - parent['children'].append(image) - image['parent'] = parent + if image.parent_name == parent_name: + parent.children.append(image) + image.parent = parent def build_queue(self, push_queue): """Organizes Queue list @@ -791,9 +849,9 @@ class KollaWorker(object): queue = six.moves.queue.Queue() for image in self.images: - if image['parent'] is None: + if image.parent is None: queue.put(BuildTask(self.conf, image, push_queue)) - LOG.debug('%s:Added image to queue', image['name']) + LOG.debug('%s:Added image to queue', image.name) return queue diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index d92906cedf..98c240d6e3 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -10,6 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import fixtures import itertools import mock @@ -20,24 +21,18 @@ from kolla.cmd import build from kolla.tests import base -FAKE_IMAGE = { - 'name': 'image-base', - 'status': 'matched', - 'parent': None, - 'parent_name': None, - 'path': '/fake/path', - 'plugins': [], - 'fullname': 'image-base:latest', -} +FAKE_IMAGE = build.Image('image-base', 'image-base:latest', + '/fake/path', parent_name=None, + parent=None, status=build.STATUS_MATCHED) class TasksTest(base.TestCase): def setUp(self): super(TasksTest, self).setUp() - self.image = FAKE_IMAGE.copy() + self.image = copy.deepcopy(FAKE_IMAGE) # NOTE(jeffrey4l): use a real, temporary dir - self.image['path'] = self.useFixture(fixtures.TempDir()).path + self.image.path = self.useFixture(fixtures.TempDir()).path @mock.patch.dict(os.environ, clear=True) @mock.patch('docker.Client') @@ -45,7 +40,7 @@ class TasksTest(base.TestCase): pusher = build.PushTask(self.conf, self.image) pusher.run() mock_client().push.assert_called_once_with( - self.image['fullname'], stream=True, insecure_registry=True) + self.image.canonical_name, stream=True, insecure_registry=True) @mock.patch.dict(os.environ, clear=True) @mock.patch('docker.Client') @@ -55,7 +50,7 @@ class TasksTest(base.TestCase): builder.run() mock_client().build.assert_called_once_with( - path=self.image['path'], tag=self.image['fullname'], + path=self.image.path, tag=self.image.canonical_name, nocache=False, rm=True, pull=True, forcerm=True, buildargs=None) @@ -74,7 +69,7 @@ class TasksTest(base.TestCase): builder.run() mock_client().build.assert_called_once_with( - path=self.image['path'], tag=self.image['fullname'], + path=self.image.path, tag=self.image.canonical_name, nocache=False, rm=True, pull=True, forcerm=True, buildargs=build_args) @@ -92,7 +87,7 @@ class TasksTest(base.TestCase): builder.run() mock_client().build.assert_called_once_with( - path=self.image['path'], tag=self.image['fullname'], + path=self.image.path, tag=self.image.canonical_name, nocache=False, rm=True, pull=True, forcerm=True, buildargs=build_args) @@ -112,7 +107,7 @@ class TasksTest(base.TestCase): builder.run() mock_client().build.assert_called_once_with( - path=self.image['path'], tag=self.image['fullname'], + path=self.image.path, tag=self.image.canonical_name, nocache=False, rm=True, pull=True, forcerm=True, buildargs=build_args) @@ -121,7 +116,7 @@ class TasksTest(base.TestCase): @mock.patch('docker.Client') @mock.patch('requests.get') def test_requests_get_timeout(self, mock_get, mock_client): - self.image['source'] = { + self.image.source = { 'source': 'http://fake/source', 'type': 'url', 'name': 'fake-image-base' @@ -129,12 +124,12 @@ class TasksTest(base.TestCase): push_queue = mock.Mock() builder = build.BuildTask(self.conf, self.image, push_queue) mock_get.side_effect = requests.exceptions.Timeout - get_result = builder.process_source(self.image, self.image['source']) + get_result = builder.process_source(self.image, self.image.source) self.assertIsNone(get_result) - self.assertEqual(self.image['status'], 'error') - self.assertEqual(self.image['logs'], str()) - mock_get.assert_called_once_with(self.image['source']['source'], + self.assertEqual(self.image.status, build.STATUS_ERROR) + self.assertEqual(str(self.image.logs), str()) + mock_get.assert_called_once_with(self.image.source['source'], timeout=120) self.assertFalse(builder.success) @@ -146,8 +141,8 @@ class KollaWorkerTest(base.TestCase): def setUp(self): super(KollaWorkerTest, self).setUp() - image = FAKE_IMAGE.copy() - image['status'] = None + image = copy.deepcopy(FAKE_IMAGE) + image.status = None self.images = [image] def test_supported_base_type(self): @@ -188,14 +183,15 @@ class KollaWorkerTest(base.TestCase): 'type': 'git' } for image in kolla.images: - if image['name'] == 'neutron-server': - self.assertEqual(image['plugins'][0], expected_plugin) + if image.name == 'neutron-server': + self.assertEqual(image.plugins[0], expected_plugin) break else: self.fail('Can not find the expected neutron arista plugin') def _get_matched_images(self, images): - return [image for image in images if image['status'] == 'matched'] + return [image for image in images + if image.status == build.STATUS_MATCHED] def test_without_profile(self): kolla = build.KollaWorker(self.conf)