Drop SkopeoImageUploader
This removes the skopeo based container image uploader which was deprecated back in stein. It doesn't work with our image-serve registry so let's drop this code and dependency. Change-Id: I23157ad3a96553047a52f10ed203f71366da49ef
This commit is contained in:
Alex Schultz
James Slagle
parent
15c41eaa46
commit
d70280eb99
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
other:
|
||||
- |
|
||||
Removed skopeo based container image uploader. This has been deprecated
|
||||
since stein and does not work with our current registry implementation.
|
||||
|
|
@ -466,7 +466,6 @@ class ImageUploadManager(BaseImageManager):
|
|||
config_files = []
|
||||
super(ImageUploadManager, self).__init__(config_files)
|
||||
self.uploaders = {
|
||||
'skopeo': SkopeoImageUploader(),
|
||||
'python': PythonImageUploader()
|
||||
}
|
||||
self.uploaders['python'].init_global_state(lock)
|
||||
|
|
@ -1275,194 +1274,6 @@ class BaseImageUploader(object):
|
|||
LOG.debug('%s %s' % (r.status_code, r.reason))
|
||||
|
||||
|
||||
class SkopeoImageUploader(BaseImageUploader):
|
||||
"""Upload images using skopeo copy"""
|
||||
|
||||
def upload_image(self, task):
|
||||
t = task
|
||||
LOG.info('[%s] Got imagename' % t.image_name)
|
||||
|
||||
source_image_local_url = parse.urlparse('containers-storage:%s'
|
||||
% t.source_image)
|
||||
|
||||
target_image_local_url = parse.urlparse('containers-storage:%s' %
|
||||
t.target_image)
|
||||
|
||||
target_username, target_password = self.credentials_for_registry(
|
||||
t.target_image_url.netloc)
|
||||
target_session = self.authenticate(
|
||||
t.target_image_url,
|
||||
username=target_username,
|
||||
password=target_password
|
||||
)
|
||||
|
||||
image_exists = False
|
||||
try:
|
||||
image_exists = self._image_exists(t.target_image,
|
||||
target_session)
|
||||
except Exception:
|
||||
LOG.warning('[%s] Failed to check if the target '
|
||||
'image exists' % t.target_image)
|
||||
pass
|
||||
if t.modify_role and image_exists:
|
||||
LOG.warning('[%s] Skipping upload for modified '
|
||||
'image' % t.target_image)
|
||||
target_session.close()
|
||||
return []
|
||||
|
||||
# Keep the target session open yet
|
||||
source_username, source_password = self.credentials_for_registry(
|
||||
t.source_image_url.netloc)
|
||||
source_session = self.authenticate(
|
||||
t.source_image_url,
|
||||
username=source_username,
|
||||
password=source_password
|
||||
)
|
||||
try:
|
||||
source_inspect = self._inspect(
|
||||
t.source_image_url,
|
||||
session=source_session)
|
||||
source_layers = source_inspect.get('Layers', [])
|
||||
self._cross_repo_mount(
|
||||
t.target_image_url, self.image_layers, source_layers,
|
||||
session=target_session)
|
||||
except Exception:
|
||||
LOG.error('[%s] Failed uploading the target '
|
||||
'image' % t.target_image)
|
||||
raise
|
||||
finally:
|
||||
source_session.close()
|
||||
target_session.close()
|
||||
|
||||
to_cleanup = []
|
||||
|
||||
if t.modify_role:
|
||||
|
||||
# Copy from source registry to local storage
|
||||
self._copy(
|
||||
t.source_image_url,
|
||||
source_image_local_url,
|
||||
)
|
||||
if t.cleanup in (CLEANUP_FULL, CLEANUP_PARTIAL):
|
||||
to_cleanup = [t.source_image]
|
||||
|
||||
self.run_modify_playbook(
|
||||
t.modify_role, t.modify_vars, t.source_image,
|
||||
t.target_image_source_tag, t.append_tag,
|
||||
container_build_tool='buildah')
|
||||
if t.cleanup == CLEANUP_FULL:
|
||||
to_cleanup.append(t.target_image)
|
||||
|
||||
# Copy from local storage to target registry
|
||||
self._copy(
|
||||
target_image_local_url,
|
||||
t.target_image_url,
|
||||
)
|
||||
LOG.warning('[%s] Completed modify and upload for '
|
||||
'image' % t.image_name)
|
||||
else:
|
||||
self._copy(
|
||||
t.source_image_url,
|
||||
t.target_image_url,
|
||||
)
|
||||
LOG.warning('[%s] Completed upload for image' % t.image_name)
|
||||
for layer in source_layers:
|
||||
self.image_layers.setdefault(layer, t.target_image_url)
|
||||
return to_cleanup
|
||||
|
||||
@classmethod
|
||||
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
|
||||
reraise=True,
|
||||
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
|
||||
stop=tenacity.stop_after_attempt(5)
|
||||
)
|
||||
def _copy(cls, source_url, target_url):
|
||||
source = source_url.geturl()
|
||||
target = target_url.geturl()
|
||||
LOG.info('Copying from %s to %s' % (source, target))
|
||||
cmd = ['skopeo', 'copy']
|
||||
|
||||
if source_url.netloc in [cls.insecure_registries,
|
||||
cls.no_verify_registries]:
|
||||
cmd.append('--src-tls-verify=false')
|
||||
|
||||
if target_url.netloc in [cls.insecure_registries,
|
||||
cls.no_verify_registries]:
|
||||
cmd.append('--dest-tls-verify=false')
|
||||
|
||||
cmd.append(source)
|
||||
cmd.append(target)
|
||||
LOG.info('Running %s' % ' '.join(cmd))
|
||||
env = os.environ.copy()
|
||||
try:
|
||||
process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
|
||||
out, err = process.communicate()
|
||||
LOG.info(out)
|
||||
if process.returncode != 0:
|
||||
raise ImageUploaderException('Error copying image:\n%s\n%s' %
|
||||
(' '.join(cmd), err))
|
||||
except KeyboardInterrupt:
|
||||
raise Exception('Action interrupted with ctrl+c')
|
||||
return out
|
||||
|
||||
def _delete(self, image_url, session=None):
|
||||
insecure = self.is_insecure_registry(registry_host=image_url.netloc)
|
||||
image = image_url.geturl()
|
||||
LOG.info('[%s] Deleting image' % image)
|
||||
cmd = ['skopeo', 'delete']
|
||||
|
||||
if insecure:
|
||||
cmd.append('--tls-verify=false')
|
||||
cmd.append(image)
|
||||
LOG.info('Running %s' % ' '.join(cmd))
|
||||
env = os.environ.copy()
|
||||
try:
|
||||
process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
|
||||
out, err = process.communicate()
|
||||
LOG.info(out.decode('utf-8'))
|
||||
if process.returncode != 0:
|
||||
raise ImageUploaderException('Error deleting image:\n%s\n%s' %
|
||||
(' '.join(cmd), err))
|
||||
except KeyboardInterrupt:
|
||||
raise Exception('Action interrupted with ctrl+c')
|
||||
return out
|
||||
|
||||
def cleanup(self, local_images):
|
||||
if not local_images:
|
||||
return []
|
||||
|
||||
for image in sorted(local_images):
|
||||
if not image:
|
||||
continue
|
||||
LOG.warning('[%s] Removing local copy of image' % image)
|
||||
image_url = parse.urlparse('containers-storage:%s' % image)
|
||||
self._delete(image_url)
|
||||
|
||||
def run_tasks(self):
|
||||
if not self.upload_tasks:
|
||||
return
|
||||
local_images = []
|
||||
|
||||
# Pull a single image first, to avoid duplicate pulls of the
|
||||
# same base layers
|
||||
local_images.extend(upload_task(args=self.upload_tasks.pop()))
|
||||
|
||||
# workers will be half the CPU count, to a minimum of 2
|
||||
workers = max(2, (processutils.get_worker_count() - 1))
|
||||
with futures.ThreadPoolExecutor(max_workers=workers) as p:
|
||||
for result in p.map(upload_task, self.upload_tasks):
|
||||
local_images.extend(result)
|
||||
LOG.info('result %s' % local_images)
|
||||
|
||||
# Do cleanup after all the uploads so common layers don't get deleted
|
||||
# repeatedly
|
||||
self.cleanup(local_images)
|
||||
|
||||
|
||||
class PythonImageUploader(BaseImageUploader):
|
||||
"""Upload images using a direct implementation of the registry API"""
|
||||
|
||||
|
|
|
|||
|
|
@ -17,16 +17,13 @@ import hashlib
|
|||
import io
|
||||
import json
|
||||
import operator
|
||||
import os
|
||||
import requests
|
||||
from requests_mock.contrib import fixture as rm_fixture
|
||||
import six
|
||||
from six.moves.urllib.parse import urlparse
|
||||
import tempfile
|
||||
from unittest import mock
|
||||
import zlib
|
||||
|
||||
from oslo_concurrency import processutils
|
||||
from tripleo_common.image.exception import ImageNotFoundException
|
||||
from tripleo_common.image.exception import ImageRateLimitedException
|
||||
from tripleo_common.image.exception import ImageUploaderException
|
||||
|
|
@ -364,11 +361,6 @@ class TestImageUploadManager(base.TestCase):
|
|||
uploader = manager.get_uploader('python')
|
||||
assert isinstance(uploader, image_uploader.PythonImageUploader)
|
||||
|
||||
def test_get_uploader_skopeo(self):
|
||||
manager = image_uploader.ImageUploadManager(self.filelist)
|
||||
uploader = manager.get_uploader('skopeo')
|
||||
assert isinstance(uploader, image_uploader.SkopeoImageUploader)
|
||||
|
||||
def test_get_builder_unknown(self):
|
||||
manager = image_uploader.ImageUploadManager(self.filelist)
|
||||
self.assertRaises(ImageUploaderException,
|
||||
|
|
@ -1284,257 +1276,6 @@ class TestBaseImageUploader(base.TestCase):
|
|||
)
|
||||
|
||||
|
||||
class TestSkopeoImageUploader(base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSkopeoImageUploader, self).setUp()
|
||||
self.uploader = image_uploader.SkopeoImageUploader()
|
||||
# pylint: disable=no-member
|
||||
self.uploader._copy.retry.sleep = mock.Mock()
|
||||
# pylint: disable=no-member
|
||||
self.uploader._inspect.retry.sleep = mock.Mock()
|
||||
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'RegistrySessionHelper.check_status')
|
||||
@mock.patch('os.environ')
|
||||
@mock.patch('subprocess.Popen')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader._inspect')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader.authenticate')
|
||||
def test_upload_image(self, mock_auth, mock_inspect,
|
||||
mock_popen, mock_environ, check_status):
|
||||
mock_process = mock.Mock()
|
||||
mock_process.communicate.return_value = ('copy complete', '')
|
||||
mock_process.returncode = 0
|
||||
mock_popen.return_value = mock_process
|
||||
mock_environ.copy.return_value = {}
|
||||
mock_inspect.return_value = {}
|
||||
|
||||
image = 'docker.io/t/nova-api'
|
||||
tag = 'latest'
|
||||
push_destination = 'localhost:8787'
|
||||
|
||||
self.assertEqual(
|
||||
[],
|
||||
self.uploader.upload_image(image_uploader.UploadTask(
|
||||
image + ':' + tag,
|
||||
None,
|
||||
push_destination,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
'full',
|
||||
False)
|
||||
)
|
||||
)
|
||||
mock_popen.assert_called_once_with([
|
||||
'skopeo',
|
||||
'copy',
|
||||
'docker://docker.io/t/nova-api:latest',
|
||||
'docker://localhost:8787/t/nova-api:latest'],
|
||||
env={}, stdout=-1, universal_newlines=True
|
||||
)
|
||||
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader.authenticate')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader._inspect')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'SkopeoImageUploader._copy')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader._image_exists')
|
||||
@mock.patch('tripleo_common.utils.'
|
||||
'ansible.run_ansible_playbook', autospec=True)
|
||||
def test_modify_upload_image(self, mock_ansible, mock_exists, mock_copy,
|
||||
mock_inspect, mock_auth):
|
||||
mock_exists.return_value = False
|
||||
mock_inspect.return_value = {}
|
||||
with tempfile.NamedTemporaryFile(delete=False) as logfile:
|
||||
self.addCleanup(os.remove, logfile.name)
|
||||
mock_ansible.return_value.run.return_value = {
|
||||
'log_path': logfile.name
|
||||
}
|
||||
|
||||
image = 'docker.io/t/nova-api'
|
||||
tag = 'latest'
|
||||
append_tag = 'modify-123'
|
||||
push_destination = 'localhost:8787'
|
||||
push_image = 'localhost:8787/t/nova-api'
|
||||
playbook = [{
|
||||
'tasks': [{
|
||||
'import_role': {
|
||||
'name': 'add-foo-plugin'
|
||||
},
|
||||
'name': 'Import role add-foo-plugin',
|
||||
'vars': {
|
||||
'target_image': '%s:%s' % (push_image, tag),
|
||||
'modified_append_tag': append_tag,
|
||||
'source_image': '%s:%s' % (image, tag),
|
||||
'foo_version': '1.0.1',
|
||||
'container_build_tool': 'buildah'
|
||||
}
|
||||
}],
|
||||
'hosts': 'localhost',
|
||||
'gather_facts': 'no'
|
||||
}]
|
||||
|
||||
# test response for a partial cleanup
|
||||
self.assertEqual(
|
||||
['docker.io/t/nova-api:latest'],
|
||||
self.uploader.upload_image(image_uploader.UploadTask(
|
||||
image + ':' + tag,
|
||||
None,
|
||||
push_destination,
|
||||
append_tag,
|
||||
'add-foo-plugin',
|
||||
{'foo_version': '1.0.1'},
|
||||
'partial',
|
||||
False)
|
||||
)
|
||||
)
|
||||
|
||||
mock_inspect.assert_has_calls([
|
||||
mock.call(urlparse(
|
||||
'docker://docker.io/t/nova-api:latest'
|
||||
), session=mock.ANY)
|
||||
])
|
||||
mock_copy.assert_has_calls([
|
||||
mock.call(
|
||||
urlparse('docker://docker.io/t/nova-api:latest'),
|
||||
urlparse('containers-storage:docker.io/t/nova-api:latest')
|
||||
),
|
||||
mock.call(
|
||||
urlparse('containers-storage:localhost:8787/'
|
||||
't/nova-api:latestmodify-123'),
|
||||
urlparse('docker://localhost:8787/'
|
||||
't/nova-api:latestmodify-123')
|
||||
)
|
||||
])
|
||||
mock_ansible.assert_called_once_with(
|
||||
playbook=playbook,
|
||||
work_dir=mock.ANY,
|
||||
verbosity=1,
|
||||
extra_env_variables=mock.ANY,
|
||||
override_ansible_cfg=(
|
||||
"[defaults]\n"
|
||||
"stdout_callback=tripleo_dense\n"
|
||||
"log_path="
|
||||
"/var/log/tripleo-container-image-prepare-ansible.log\n"
|
||||
)
|
||||
)
|
||||
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader.authenticate')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader._inspect')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'SkopeoImageUploader._copy')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader._image_exists')
|
||||
@mock.patch('tripleo_common.utils.'
|
||||
'ansible.run_ansible_playbook', autospec=True)
|
||||
def test_modify_image_failed(self, mock_ansible, mock_exists, mock_copy,
|
||||
mock_inspect, mock_auth):
|
||||
mock_exists.return_value = False
|
||||
mock_inspect.return_value = {}
|
||||
|
||||
image = 'docker.io/t/nova-api'
|
||||
tag = 'latest'
|
||||
append_tag = 'modify-123'
|
||||
push_destination = 'localhost:8787'
|
||||
error = processutils.ProcessExecutionError(
|
||||
'', 'ouch', -1, 'ansible-playbook')
|
||||
mock_ansible.side_effect = error
|
||||
|
||||
self.assertRaises(
|
||||
ImageUploaderException,
|
||||
self.uploader.upload_image, image_uploader.UploadTask(
|
||||
image + ':' + tag, None, push_destination,
|
||||
append_tag, 'add-foo-plugin', {'foo_version': '1.0.1'},
|
||||
'full', False)
|
||||
)
|
||||
|
||||
mock_copy.assert_called_once_with(
|
||||
urlparse('docker://docker.io/t/nova-api:latest'),
|
||||
urlparse('containers-storage:docker.io/t/nova-api:latest')
|
||||
)
|
||||
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader.authenticate')
|
||||
@mock.patch('tripleo_common.image.image_uploader.'
|
||||
'BaseImageUploader._inspect')
|
||||
@mock.patch('tripleo_common.utils.'
|
||||
'ansible.run_ansible_playbook', autospec=True)
|
||||
def test_modify_image_existing(self, mock_ansible, mock_inspect,
|
||||
mock_auth):
|
||||
mock_inspect.return_value = {'Digest': 'a'}
|
||||
|
||||
image = 'docker.io/t/nova-api'
|
||||
tag = 'latest'
|
||||
append_tag = 'modify-123'
|
||||
push_destination = 'localhost:8787'
|
||||
|
||||
result = self.uploader.upload_image(image_uploader.UploadTask(
|
||||
image + ':' + tag,
|
||||
None,
|
||||
push_destination,
|
||||
append_tag,
|
||||
'add-foo-plugin',
|
||||
{'foo_version': '1.0.1'},
|
||||
'full',
|
||||
False)
|
||||
)
|
||||
|
||||
mock_ansible.assert_not_called()
|
||||
|
||||
self.assertEqual([], result)
|
||||
|
||||
@mock.patch('os.environ')
|
||||
@mock.patch('subprocess.Popen')
|
||||
def test_copy_retry(self, mock_popen, mock_environ):
|
||||
mock_success = mock.Mock()
|
||||
mock_success.communicate.return_value = ('copy complete', '')
|
||||
mock_success.returncode = 0
|
||||
|
||||
mock_failure = mock.Mock()
|
||||
mock_failure.communicate.return_value = ('', 'ouch')
|
||||
mock_failure.returncode = 1
|
||||
mock_popen.side_effect = [
|
||||
mock_failure,
|
||||
mock_failure,
|
||||
mock_failure,
|
||||
mock_failure,
|
||||
mock_success
|
||||
]
|
||||
mock_environ.copy.return_value = {}
|
||||
|
||||
source = urlparse('docker://docker.io/t/nova-api')
|
||||
target = urlparse('containers_storage:docker.io/t/nova-api')
|
||||
|
||||
self.uploader._copy(source, target)
|
||||
|
||||
self.assertEqual(mock_failure.communicate.call_count, 4)
|
||||
self.assertEqual(mock_success.communicate.call_count, 1)
|
||||
|
||||
@mock.patch('os.environ')
|
||||
@mock.patch('subprocess.Popen')
|
||||
def test_copy_retry_failure(self, mock_popen, mock_environ):
|
||||
mock_failure = mock.Mock()
|
||||
mock_failure.communicate.return_value = ('', 'ouch')
|
||||
mock_failure.returncode = 1
|
||||
mock_popen.return_value = mock_failure
|
||||
mock_environ.copy.return_value = {}
|
||||
|
||||
source = urlparse('docker://docker.io/t/nova-api')
|
||||
target = urlparse('containers_storage:docker.io/t/nova-api')
|
||||
|
||||
self.assertRaises(
|
||||
ImageUploaderException, self.uploader._copy, source, target)
|
||||
|
||||
self.assertEqual(mock_failure.communicate.call_count, 5)
|
||||
|
||||
|
||||
class TestPythonImageUploader(base.TestCase):
|
||||
|
||||
# pylint: disable=no-member
|
||||
|
|
|
|||
Loading…
Reference in New Issue