Added packaging driver to build RPM by using mock

Option 'cache_dir' uses to specify directory where
will be downloaded remote files
The packaging controller allows to use files which
are available via HTTP as source or spec file
Each driver has its own section in input data,
this allows to use same input data for several drivers.

Change-Id: I1fb3b08fe305c3413e5aa4a9213762208a2479da
This commit is contained in:
Bulat Gaifullin 2016-07-01 15:27:18 +03:00
parent 5c9d32f234
commit cae6df70bf
15 changed files with 505 additions and 21 deletions

View File

@ -16,6 +16,9 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import tempfile
from packetary.library.connections import ConnectionsManager
from packetary.library.executor import AsynchronousSection
@ -25,7 +28,7 @@ class Configuration(object):
def __init__(self, http_proxy=None, https_proxy=None,
retries_num=0, retry_interval=0, threads_num=0,
ignore_errors_num=0):
ignore_errors_num=0, cache_dir=None):
"""Initialises.
:param http_proxy: the url of proxy for connections over http,
@ -37,6 +40,8 @@ class Configuration(object):
:param threads_num: the max number of active threads
:param ignore_errors_num: the number of errors that may occurs
before stop processing
:param cache_dir: the path to directory were will be downloaded
remote files
"""
self.http_proxy = http_proxy
@ -45,6 +50,7 @@ class Configuration(object):
self.retries_num = retries_num
self.retry_interval = retry_interval
self.threads_num = threads_num
self.cache_dir = cache_dir
class Context(object):
@ -63,12 +69,22 @@ class Context(object):
)
self._threads_num = config.threads_num
self._ignore_errors_num = config.ignore_errors_num
if config.cache_dir:
self._cache_dir = config.cache_dir
else:
self._cache_dir = os.path.join(
tempfile.gettempdir(), 'packetary-cache'
)
@property
def connection(self):
"""Gets the connection."""
return self._connection
@property
def cache_dir(self):
return self._cache_dir
def async_section(self, ignore_errors_num=None):
"""Gets the execution scope.

View File

@ -76,6 +76,12 @@ class Application(app.App):
metavar="https://username:password@proxy_host:proxy_port",
help="The URL of https proxy."
)
parser.add_argument(
"--cache-dir",
default=None,
metavar="PATH",
help="The path to the directory which be used for cache."
)
return parser

View File

@ -17,10 +17,13 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import logging
import os
import six
import stevedore
from packetary.library import utils
logger = logging.getLogger(__package__)
urljoin = six.moves.urllib.parse.urljoin
@ -66,5 +69,19 @@ class PackagingController(object):
:param output_dir: directory for new packages
:param consumer: callable, that will be called for each built package
"""
# TODO(bgaifullin) Add downloading sources and specs from URL
return self.driver.build_packages(data, output_dir, consumer)
cache = {}
with self.context.async_section() as section:
for url in self.driver.get_for_caching(data):
section.execute(self._add_to_cache, url, cache)
return self.driver.build_packages(data, cache, output_dir, consumer)
def _add_to_cache(self, url, cache):
path = utils.get_path_from_url(url, ensure_file=False)
if not utils.is_local(url):
path = os.path.join(
self.context.cache_dir, utils.get_filename_from_uri(path)
)
self.context.connection.retrieve(url, path)
cache[url] = path

View File

@ -134,11 +134,15 @@ class PackagingDriverBase(object):
"""Gets the json-schema to validate input data."""
@abc.abstractmethod
def build_packages(self, data, output_dir, consumer):
def get_for_caching(self, data):
"""Gets the list of url(s), that should be added to cache."""
@abc.abstractmethod
def build_packages(self, data, cache, output_dir, consumer):
"""Build package from sources.
:param data: the input data for building packages,
the format of data depends on selected driver
:param data: the input data
:param cache: the cache instance with resources, which is downloaded
:param output_dir: directory for new packages
:param consumer: callable, that will be called for each built package
"""

View File

@ -0,0 +1,119 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import glob
import os
import subprocess
from packetary.drivers.base import PackagingDriverBase
from packetary.library import utils
from packetary.schemas import RPM_PACKAGING_SCHEMA
class MockDriver(PackagingDriverBase):
def __init__(self, config_file):
super(MockDriver, self).__init__()
self.mock_bin = utils.find_executable('mock')
if config_file:
self.config_dir = os.path.dirname(config_file)
self.config_name = os.path.splitext(
os.path.basename(config_file)
)[0]
else:
self.config_dir = ''
self.config_name = ''
def get_data_schema(self):
return RPM_PACKAGING_SCHEMA
def get_for_caching(self, data):
return [data['src'], data['rpm']['spec']]
def build_packages(self, data, cache, output_dir, consumer):
src = cache[data['src']]
spec = cache[data['rpm']['spec']]
options = data['rpm'].get('options', {})
with utils.create_tmp_dir() as tmpdir:
self._buildsrpm(
resultdir=tmpdir, spec=spec, sources=src, **options
)
srpms_dir = os.path.join(output_dir, 'SRPM')
utils.ensure_dir_exist(srpms_dir)
srpms = glob.iglob(os.path.join(srpms_dir, '*.src.rpm'))
rpms_dir = os.path.join(output_dir, 'RPM')
utils.ensure_dir_exist(rpms_dir)
self._rebuild(srpms, resultdir=tmpdir, **options)
# rebuild commands rebuilds source rpm too
# notify only about last version
for rpm in utils.move_files(tmpdir, srpms_dir, '*.src.rpm'):
consumer(rpm)
for rpm in utils.move_files(tmpdir, rpms_dir, '*.rpm'):
consumer(rpm)
def _buildsrpm(self, spec, sources, **kwargs):
"""Builds the specified SRPM either from a spec file.
:param spec: Specifies spec file to use to build an SRPM
:param sources: Specifies sources (either a single file or a directory
of files)to use to build an SRPM
:kwargs: the other mock parameters, for details see `man mock`
"""
self.logger.info("buildsrpm '%s' '%s'", spec, sources)
return self._invoke_mock(
'buildsrpm', spec=spec, sources=sources, **kwargs
)
def _rebuild(self, srpms, **kwargs):
"""Rebuilds the specified SRPM(s).
:param srpms: The list of SRPM(s) for rebuilding.
:kwargs: the other mock parameters, for details see `man mock`
"""
self.logger.info("rebuild %s", srpms)
return self._invoke_mock('rebuild', *srpms, **kwargs)
def _invoke_mock(self, command, *args, **kwargs):
cmdline = self._assemble_cmdline(command, args, kwargs)
self.logger.debug("start command: '%'", ' '.join(cmdline))
subprocess.check_call(cmdline)
def _assemble_cmdline(self, command, args, kwargs):
def add_option(name, value):
if isinstance(value, list):
for item in value:
add_option(name, item)
else:
cmd.append('--' + name)
cmd.append(value)
cmd = [self.mock_bin]
if self.config_name:
add_option('root', self.config_name)
if self.config_dir:
add_option('configdir', self.config_dir)
for k, v in kwargs.items():
add_option(k, v)
cmd.append('--' + command)
cmd.extend(args)
return cmd

View File

@ -18,8 +18,13 @@
from __future__ import with_statement
import contextlib
from distutils import spawn
import errno
import glob
import os
import shutil
import tempfile
import six
@ -73,6 +78,12 @@ def get_size_and_checksum_for_files(files, checksum_algo):
yield filename, size, checksum
def is_local(url):
"""Checks that url reflects local path."""
comps = urlparse(url, scheme="file")
return comps.scheme == "file"
def get_path_from_url(url, ensure_file=True):
"""Get the path from the URL.
@ -135,3 +146,48 @@ def ensure_dir_exist(path):
except OSError as e:
if e.errno != errno.EEXIST:
raise
def find_executable(name, __finder=spawn.find_executable):
"""Finds executable by name in directories listed in 'path'."""
path = __finder(name)
if not path:
raise RuntimeError(
"{0} does not found in directories listed in 'path'."
.format(name)
)
return path
@contextlib.contextmanager
def create_tmp_dir():
"""Creates temporary directory.
The directory will be removed automatically on exit from context
:return: path of directory that has been created
"""
tmpdir = tempfile.mkdtemp()
try:
yield tmpdir
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
def move_files(src, dst, pattern='*'):
"""Moves files by pattern from directory src to directory dst.
:param src: the source directory path
:param dst: the destination directory path
:param pattern: the pattern to search files in directory
:return: the list of files that has been moved
"""
files = []
for f in glob.iglob(os.path.join(src, pattern)):
dst_path = os.path.join(dst, os.path.basename(f))
shutil.move(f, dst_path)
files.append(dst_path)
return files

View File

@ -19,11 +19,13 @@
from packetary.schemas.deb_repo_schema import DEB_REPO_SCHEMA
from packetary.schemas.package_files_schema import PACKAGE_FILES_SCHEMA
from packetary.schemas.requirements_schema import REQUIREMENTS_SCHEMA
from packetary.schemas.rpm_packaging_schema import RPM_PACKAGING_SCHEMA
from packetary.schemas.rpm_repo_schema import RPM_REPO_SCHEMA
__all__ = [
"DEB_REPO_SCHEMA",
"PACKAGE_FILES_SCHEMA",
"REQUIREMENTS_SCHEMA",
"RPM_PACKAGING_SCHEMA",
"RPM_REPO_SCHEMA",
]

View File

@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
RPM_PACKAGING_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"required": ["src", "rpm"],
"properties": {
"src": {"type": "string"},
"rpm": {
"type": "object",
"required": ["spec"],
"properties": {
"spec": {"type": "string"},
"options": {
"type": "object",
"patternProperties": {
"^[a-zA-Z][0-9a-z_-]*$": {
'anyOf': [{"type": "array"}, {"type": "string"}]
}
}
}
}
}
}
}

View File

@ -32,7 +32,8 @@ class TestContext(base.TestCase):
retries_num=5,
retry_interval=10,
http_proxy="http://localhost",
https_proxy="https://localhost"
https_proxy="https://localhost",
cache_dir="/root/cache"
)
@mock.patch("packetary.api.context.ConnectionsManager")
@ -55,3 +56,12 @@ class TestContext(base.TestCase):
self.assertIs(s, async_section())
ctx.async_section(0)
async_section.assert_called_with(2, 0)
@mock.patch("packetary.api.context.tempfile")
def test_cache_dir(self, tempfile_mock):
ctx = context.Context(self.config)
self.assertEqual(self.config.cache_dir, ctx.cache_dir)
self.config.cache_dir = None
tempfile_mock.gettempdir.return_value = '/tmp'
ctx2 = context.Context(self.config)
self.assertEqual('/tmp/packetary-cache', ctx2.cache_dir)

View File

@ -124,3 +124,40 @@ class TestLibraryUtils(base.TestCase):
("", ("file:///root/",))
]
self._check_cases(self.assertEqual, cases, utils.get_filename_from_uri)
def test_is_local(self):
self.assertTrue(utils.is_local("/root/1.txt"))
self.assertTrue(utils.is_local("file:///root/1.txt"))
self.assertTrue(utils.is_local("./root/1.txt"))
self.assertFalse(utils.is_local("http://localhost/root/1.txt"))
def test_find_executable(self):
finder = mock.MagicMock(side_effect=['/bin/test', None])
self.assertEqual(
'/bin/test', utils.find_executable('test', __finder=finder)
)
self.assertRaises(
RuntimeError, utils.find_executable, 'test2', __finder=finder
)
@mock.patch.multiple(
"packetary.library.utils", tempfile=mock.DEFAULT, shutil=mock.DEFAULT
)
def test_create_tmp_dir(self, tempfile, shutil):
with utils.create_tmp_dir() as tmpdir:
self.assertIs(tempfile.mkdtemp.return_value, tmpdir)
tempfile.mkdtemp.assert_called_once_with()
shutil.rmtree.assert_called_once_with(tmpdir, ignore_errors=True)
@mock.patch("packetary.library.utils.shutil")
@mock.patch("packetary.library.utils.glob")
def test_move_files(self, glob_mock, shutil_mock):
glob_mock.iglob.return_value = ["d1/f1", "d1/f2"]
files = utils.move_files("d1", "d2", "*.*")
shutil_mock.move.assert_has_calls(
[mock.call("d1/f1", "d2/f1"),
mock.call("d1/f2", "d2/f2")],
any_order=False
)
self.assertEqual(["d2/f1", "d2/f2"], files)

View File

@ -0,0 +1,109 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import mock
from packetary.drivers import mock_driver
from packetary.schemas import RPM_PACKAGING_SCHEMA
from packetary.tests import base
class TestMockDriver(base.TestCase):
def setUp(self):
with mock.patch('packetary.drivers.mock_driver.utils') as u_mock:
u_mock.find_executable.return_value = '/bin/mock'
self.driver = mock_driver.MockDriver('/etc/mock/default.cfg')
self.driver.logger = mock.MagicMock()
def test_get_data_schema(self):
self.assertIs(RPM_PACKAGING_SCHEMA, self.driver.get_data_schema())
def get_for_caching(self):
data = {'src': '/src', 'rpm': {'spec': '/spec'}}
self.assertEqual(['/src', '/spec'], self.driver.get_for_caching(data))
@mock.patch('packetary.drivers.mock_driver.utils')
@mock.patch('packetary.drivers.mock_driver.glob')
def test_build_packages(self, glob_mock, utils_mock):
packages = []
expected_packages = ['/tmp/package1.srpm', '/tmp/package1.rpm']
utils_mock.move_files.side_effect = [
expected_packages[:1], expected_packages[1:]
]
glob_mock.iglob.return_value = expected_packages[:1]
utils_mock.create_tmp_dir().__enter__.return_value = '/tmp'
data = {'src': '/src', 'rpm': {'spec': '/spec', 'options': {'a': '1'}}}
cache = {'/src': '/src', '/spec': '/spec'}
with mock.patch.object(self.driver, '_invoke_mock') as call_mock:
self.driver.build_packages(data, cache, '/tmp', packages.append)
self.assertEqual(expected_packages, packages)
utils_mock.create_tmp_dir.assert_called_with()
utils_mock.create_tmp_dir().__enter__.assert_called_once_with()
utils_mock.create_tmp_dir().__exit__.assert_called_once_with(
None, None, None
)
utils_mock.ensure_dir_exist.assert_has_calls(
[mock.call('/tmp/SRPM'), mock.call('/tmp/RPM')],
)
tmpdir = utils_mock.create_tmp_dir().__enter__()
call_mock.assert_has_calls([
mock.call(
'buildsrpm', resultdir=tmpdir, spec='/spec',
sources='/src', a='1'
),
mock.call('rebuild', expected_packages[0], resultdir=tmpdir, a='1')
])
utils_mock.move_files.assert_has_calls(
[mock.call(tmpdir, '/tmp/SRPM', '*.src.rpm'),
mock.call(tmpdir, '/tmp/RPM', '*.rpm')]
)
@mock.patch('packetary.drivers.mock_driver.subprocess')
def test_invoke_mock(self, subprocess_mock):
with mock.patch.object(self.driver, '_assemble_cmdline') as _assemble:
self.driver._invoke_mock('cmd', 'arg', key1='1')
_assemble.assert_called_once_with('cmd', ('arg', ), {'key1': '1'})
subprocess_mock.check_call.assert_called_once_with(
_assemble.return_value
)
def test_assemble_cmdline(self):
self.assertEqual(
[
'/bin/mock', '--root', 'default', '--configdir', '/etc/mock',
'--src', '/src', '--rebuild', 'package1'
],
self.driver._assemble_cmdline(
'rebuild', ('package1',), {'src': '/src'}
)
)
self.driver.config_dir = None
self.assertEqual(
['/bin/mock', '--root', 'default', '--src', '/src', '--build'],
self.driver._assemble_cmdline('build', (), {'src': '/src'})
)
self.driver.config_name = None
self.assertEqual(
['/bin/mock', '--src', 'src1', '--src', 'src2', '--build'],
self.driver._assemble_cmdline(
'build', (), {'src': ['src1', 'src2']}
)
)

View File

@ -22,13 +22,17 @@ from packetary.controllers import PackagingController
from packetary.drivers.base import PackagingDriverBase
from packetary.tests import base
from packetary.tests.stubs.executor import Executor
class TestPackagingController(base.TestCase):
def setUp(self):
super(TestPackagingController, self).setUp()
self.context = mock.MagicMock()
self.context.cache_dir = '/root'
self.context.async_section.return_value = Executor()
self.driver = mock.MagicMock(spec=PackagingDriverBase)
self.controller = PackagingController("contex", self.driver)
self.controller = PackagingController(self.context, self.driver)
@mock.patch("packetary.controllers.packaging.stevedore")
def test_load_fail_if_unknown_driver(self, stevedore):
@ -61,10 +65,27 @@ class TestPackagingController(base.TestCase):
self.driver.get_data_schema.assert_called_once_with()
def test_build_packages(self):
data = {'sources': '/sources'}
src = '/src'
spec = 'http://localhost/spec.txt'
data = {'src': src, 'test': {'spec': spec}}
self.driver.get_for_caching.return_value = [src, spec]
output_dir = '/tmp/'
callback = mock.MagicMock()
self.controller.build_packages(data, output_dir, callback)
self.driver.build_packages.assert_called_once_with(
data, output_dir, callback
data,
{src: src, spec: '/root/spec.txt'},
output_dir,
callback
)
def test_add_to_cache(self):
cache = {}
self.controller._add_to_cache('/test', cache)
self.assertEqual('/test', cache['/test'])
self.assertEqual(0, self.context.connection.retrieve.call_count)
self.controller._add_to_cache('http://localhost/test.txt', cache)
self.assertEqual('/root/test.txt', cache['http://localhost/test.txt'])
self.context.connection.retrieve.assert_called_once_with(
'http://localhost/test.txt', '/root/test.txt'
)

View File

@ -109,7 +109,7 @@ class TestRepositoryApi(base.TestCase):
config = api.Configuration(
http_proxy="http://localhost", https_proxy="https://localhost",
retries_num=10, retry_interval=1, threads_num=8,
ignore_errors_num=6
ignore_errors_num=6, cache_dir='/tmp/cache'
)
context = api.Context(config)
api.RepositoryApi.create(context, "deb", "x86_64")

View File

@ -77,8 +77,7 @@ class TestRepositorySchemaBase(base.TestCase):
class TestDebRepoSchema(TestRepositorySchemaBase):
def setUp(self):
self.schema = schemas.DEB_REPO_SCHEMA
schema = schemas.DEB_REPO_SCHEMA
def test_valid_repo_data(self):
repo_data = {
@ -141,8 +140,7 @@ class TestDebRepoSchema(TestRepositorySchemaBase):
class TestRpmRepoSchema(TestRepositorySchemaBase):
def setUp(self):
self.schema = schemas.RPM_REPO_SCHEMA
schema = schemas.RPM_REPO_SCHEMA
def test_valid_repo_data(self):
repo_data = {
@ -171,9 +169,7 @@ class TestRpmRepoSchema(TestRepositorySchemaBase):
class TestRequirementsSchema(base.TestCase):
def setUp(self):
self.schema = schemas.REQUIREMENTS_SCHEMA
schema = schemas.REQUIREMENTS_SCHEMA
def test_valid_requirements_data(self):
requirements_data = {
@ -229,8 +225,7 @@ class TestRequirementsSchema(base.TestCase):
class TestPackageFilesSchema(base.TestCase):
def setUp(self):
self.schema = schemas.PACKAGE_FILES_SCHEMA
schema = schemas.PACKAGE_FILES_SCHEMA
def test_valid_file_urls(self):
file_urls = [
@ -272,3 +267,51 @@ class TestPackageFilesSchema(base.TestCase):
jsonschema.ValidationError, "does not match",
jsonschema.validate, url, self.schema
)
class TestRpmPackagingSchema(base.TestCase):
schema = schemas.RPM_PACKAGING_SCHEMA
def test_valid_data(self):
data = {
'src': '/sources',
'rpm': {
'spec': '/spec.txt',
'options': {'with': 'option1', 'without': ['option2']}
}
}
self.assertNotRaises(
jsonschema.ValidationError, jsonschema.validate, data, self.schema
)
def test_validation_fail_if_option_is_invalid(self):
data = {
'src': '/sources',
'rpm': {'spec': '/spec.txt', 'options': {'with': 1}}
}
self.assertRaisesRegexp(
jsonschema.ValidationError,
"1 is not valid under any of the given schemas",
jsonschema.validate, data, self.schema
)
def test_validation_spec_is_mandatory(self):
data = {'src': '/sources', 'rpm': {'options': {'with': '1'}}}
self.assertRaisesRegexp(
jsonschema.ValidationError, "'spec' is a required property",
jsonschema.validate, data, self.schema
)
def test_validation_src_is_mandatory(self):
data = {'rpm': {'spec': '/spec.txt'}}
self.assertRaisesRegexp(
jsonschema.ValidationError, "'src' is a required property",
jsonschema.validate, data, self.schema
)
def test_validation_rpm_is_mandatory(self):
data = {'src': '/sources'}
self.assertRaisesRegexp(
jsonschema.ValidationError, "'rpm' is a required property",
jsonschema.validate, data, self.schema
)

View File

@ -30,12 +30,15 @@ packages =
console_scripts =
packetary=packetary.cli.app:main
packetary.packaging_drivers =
mock=packetary.drivers.mock_driver:MockDriver
packetary.repository_drivers =
deb=packetary.drivers.deb_driver:DebRepositoryDriver
rpm=packetary.drivers.rpm_driver:RpmRepositoryDriver
packetary =
build=packetary.cli.commands.build.BuildPackageCommand
build=packetary.cli.commands.build:BuildPackageCommand
clone=packetary.cli.commands.clone:CloneCommand
create=packetary.cli.commands.create:CreateCommand
packages=packetary.cli.commands.packages:ListOfPackages