[packetary] Packages indexer

Change-Id: I981fe7ae8f0aba8314475c3633f325e18d5e3bb5
Implements: blueprint refactor-local-mirror-scripts
Partial-Bug: #1487077
This commit is contained in:
Bulat Gaifullin 2015-10-20 18:33:58 +03:00
parent 6e1b82b205
commit 6a9c463042
13 changed files with 1108 additions and 158 deletions

View File

@ -14,12 +14,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import errno
import logging
import os
import six
import six.moves.http_client as http_client
import six.moves.urllib.request as urllib_request
import six.moves.urllib_error as urllib_error
import six.moves.urllib.request as urllib
import six.moves.urllib_error as urlerror
import time
from packetary.library.streams import StreamWrapper
@ -31,11 +32,11 @@ logger = logging.getLogger(__package__)
RETRYABLE_ERRORS = (http_client.HTTPException, IOError)
class RangeError(urllib_error.URLError):
class RangeError(urlerror.URLError):
pass
class RetryableRequest(urllib_request.Request):
class RetryableRequest(urllib.Request):
offset = 0
retries_left = 1
start_time = 0
@ -66,6 +67,7 @@ class ResumableResponse(StreamWrapper):
self.request.offset += len(chunk)
return chunk
except RETRYABLE_ERRORS as e:
# TODO(check hashsums)
response = self.opener.error(
self.request.get_type(), self.request,
self.stream, 502, six.text_type(e), self.stream.info()
@ -73,7 +75,7 @@ class ResumableResponse(StreamWrapper):
self.stream = response.stream
class RetryHandler(urllib_request.BaseHandler):
class RetryHandler(urllib.BaseHandler):
"""urllib Handler to add ability for retrying on server errors."""
@staticmethod
@ -114,23 +116,36 @@ class RetryHandler(urllib_request.BaseHandler):
https_response = http_response
class Connection(object):
"""Helper class to deal with streams."""
class ConnectionsManager(object):
"""The connections manager."""
def __init__(self, opener, retries_num):
"""Initializes.
def __init__(self, proxy=None, secure_proxy=None, retries_num=0):
"""Initialises.
:param opener: the instance of urllib.OpenerDirector
:param proxy: the url of proxy for http-connections
:param secure_proxy: the url of proxy for https-connections
:param retries_num: the number of allowed retries
"""
self.opener = opener
if proxy:
proxies = {
"http": proxy,
"https": secure_proxy or proxy,
}
else:
proxies = None
self.retries_num = retries_num
self.opener = urllib.build_opener(
RetryHandler(),
urllib.ProxyHandler(proxies)
)
def make_request(self, url, offset=0):
"""Makes new http request.
:param url: the remote file`s url
:param offset: the number of bytes from begin, that will be skipped
:param offset: the number of bytes from the beginning,
that will be skipped
:return: The new http request
"""
@ -146,14 +161,15 @@ class Connection(object):
"""Opens remote file for streaming.
:param url: the remote file`s url
:param offset: the number of bytes from begin, that will be skipped
:param offset: the number of bytes from the beginning,
that will be skipped
"""
request = self.make_request(url, offset)
while 1:
try:
return self.opener.open(request)
except (RangeError, urllib_error.HTTPError):
except (RangeError, urlerror.HTTPError):
raise
except RETRYABLE_ERRORS as e:
if request.retries_left <= 0:
@ -169,7 +185,8 @@ class Connection(object):
:param url: the remote file`s url
:param filename: the file`s name, that includes path on local fs
:param offset: the number of bytes from begin, that will be skipped
:param offset: the number of bytes from the beginning,
that will be skipped
"""
self._ensure_dir_exists(filename)
@ -180,7 +197,8 @@ class Connection(object):
if offset == 0:
raise
logger.warning(
"Failed to resume download, starts from begin: %s", url
"Failed to resume download, starts from the beginning: %s",
url
)
self._copy_stream(fd, url, 0)
finally:
@ -194,7 +212,7 @@ class Connection(object):
try:
os.makedirs(target_dir)
except OSError as e:
if e.errno != 17:
if e.errno != errno.EEXIST:
raise
def _copy_stream(self, fd, url, offset):
@ -202,7 +220,8 @@ class Connection(object):
:param fd: the file`s descriptor
:param url: the remote file`s url
:param offset: the number of bytes from begin, that will be skipped
:param offset: the number of bytes from the beginning,
that will be skipped
"""
source = self.open_stream(url, offset)
@ -214,67 +233,3 @@ class Connection(object):
if not chunk:
break
os.write(fd, chunk)
class ConnectionContext(object):
"""Helper class acquire and release connection within context."""
def __init__(self, connection, on_exit):
self.connection = connection
self.on_exit = on_exit
def __enter__(self):
return self.connection
def __exit__(self, *_):
self.on_exit(self.connection)
class ConnectionsPool(object):
"""Controls the number of simultaneously opened connections."""
MIN_CONNECTIONS_COUNT = 1
def __init__(self, count=0, proxy=None, secure_proxy=None, retries_num=0):
"""Initialises.
:param count: the number of allowed simultaneously connections
:param proxy: the url of proxy for http-connections
:param secure_proxy: the url of proxy for https-connections
:param retries_num: the number of allowed retries
"""
if proxy:
proxies = {
"http": proxy,
"https": secure_proxy or proxy,
}
else:
proxies = None
opener = urllib_request.build_opener(
RetryHandler(),
urllib_request.ProxyHandler(proxies)
)
limit = max(count, self.MIN_CONNECTIONS_COUNT)
connections = six.moves.queue.Queue()
while limit > 0:
connections.put(Connection(opener, retries_num))
limit -= 1
self.free = connections
def get(self, timeout=None):
"""Gets the free connection.
Blocks in case if there is no free connections.
:param timeout: the timeout in seconds to wait.
by default infinity waiting.
"""
return ConnectionContext(
self.free.get(timeout=timeout), self._release
)
def _release(self, connection):
"""Puts back connection to free connections."""
self.free.put(connection)

View File

@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from packetary.objects.index import Index
from packetary.objects.package import FileChecksum
from packetary.objects.package import Package
from packetary.objects.package_relation import PackageRelation
from packetary.objects.package_relation import VersionRange
from packetary.objects.repository import Repository
__all__ = [
"FileChecksum",
"Index",
"Package",
"PackageRelation",
"Repository",
"VersionRange",
]

60
packetary/objects/base.py Normal file
View File

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class ComparableObject(object):
"""Superclass for objects, that should be comparable.
Note: because python3 does not support __cmp__ slot, use
cmp method to implement all of compare methods.
"""
@abc.abstractmethod
def cmp(self, other):
"""Compares with other object.
:return: value is negative if if self < other, zero if self == other
strictly positive if x > y
"""
def __lt__(self, other):
return self.cmp(other) < 0
def __le__(self, other):
return self.cmp(other) <= 0
def __gt__(self, other):
return self.cmp(other) > 0
def __ge__(self, other):
return self.cmp(other) >= 0
def __eq__(self, other):
if other is self:
return True
return isinstance(other, type(self)) and self.cmp(other) == 0
def __ne__(self, other):
if other is self:
return False
return not isinstance(other, type(self)) or self.cmp(other) != 0
def __cmp__(self, other):
return self.cmp(other)

207
packetary/objects/index.py Normal file
View File

@ -0,0 +1,207 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from bintrees import FastRBTree
from collections import defaultdict
import functools
import operator
import six
def _make_operator(direction, op):
"""Makes search operator from low-level operation and search direction."""
return functools.partial(direction, condition=op)
def _start_upperbound(versions, version, condition):
"""Gets all versions from [start, version] that meet condition.
:param versions: the tree of versions.
:param version: the required version
:param condition: condition for search
:return: the list of found versions
"""
result = list(versions.value_slice(None, version))
try:
bound = versions.ceiling_item(version)
if condition(bound[0], version):
result.append(bound[1])
except KeyError:
pass
return result
def _lowerbound_end(versions, version, condition):
"""Gets all versions from [version, end] that meet condition.
:param versions: the tree of versions.
:param version: the required version
:param condition: condition for search
:return: the list of found versions
"""
result = []
items = iter(versions.item_slice(version, None))
bound = next(items, None)
if bound is None:
return result
if condition(bound[0], version):
result.append(bound[1])
result.extend(x[1] for x in items)
return result
def _equal(tree, version):
"""Gets the package with specified version."""
if version in tree:
return [tree[version]]
return []
def _any(tree, _):
"""Gets the package with max version."""
return list(tree.values())
class Index(object):
"""The search index for packages.
Builds three search-indexes:
- index of packages with versions.
- index of virtual packages (provides).
- index of obsoleted packages (obsoletes).
Uses to find package by name and range of versions.
"""
operators = {
None: _any,
"lt": _make_operator(_start_upperbound, operator.lt),
"le": _make_operator(_start_upperbound, operator.le),
"gt": _make_operator(_lowerbound_end, operator.gt),
"ge": _make_operator(_lowerbound_end, operator.ge),
"eq": _equal,
}
def __init__(self):
self.packages = defaultdict(FastRBTree)
self.obsoletes = defaultdict(FastRBTree)
self.provides = defaultdict(FastRBTree)
def __iter__(self):
"""Iterates over all packages including versions."""
return self.get_all()
def __len__(self, _reduce=six.functools.reduce):
"""Returns the total number of packages with versions."""
return _reduce(
lambda x, y: x + len(y),
six.itervalues(self.packages),
0
)
def get_all(self):
"""Gets sequence from all of packages including versions."""
for versions in six.itervalues(self.packages):
for version in versions.values():
yield version
def find(self, name, version):
"""Finds the package by name and range of versions.
:param name: the package`s name.
:param version: the range of versions.
:return: the package if it is found, otherwise None
"""
candidates = self.find_all(name, version)
if len(candidates) > 0:
return candidates[-1]
return None
def find_all(self, name, version):
"""Finds the packages by name and range of versions.
:param name: the package`s name.
:param version: the range of versions.
:return: the list of suitable packages
"""
if name in self.packages:
candidates = self._find_versions(
self.packages[name], version
)
if len(candidates) > 0:
return candidates
if name in self.obsoletes:
return self._resolve_relation(
self.obsoletes[name], version
)
if name in self.provides:
return self._resolve_relation(
self.provides[name], version
)
return []
def add(self, package):
"""Adds new package to indexes.
:param package: the package object.
"""
self.packages[package.name][package.version] = package
key = package.name, package.version
for obsolete in package.obsoletes:
self.obsoletes[obsolete.name][key] = obsolete
for provide in package.provides:
self.provides[provide.name][key] = provide
def _resolve_relation(self, relations, version):
"""Resolve relation according to relations index.
:param relations: the index of relations
:param version: the range of versions
:return: package if found, otherwise None
"""
for key, candidate in relations.iter_items(reverse=True):
if candidate.version.has_intersection(version):
return [self.packages[key[0]][key[1]]]
return []
@staticmethod
def _find_versions(versions, version):
"""Searches accurate version.
Search for the highest version out of intersection
of existing and required range of versions.
:param versions: the existing versions
:param version: the required range of versions
:return: package if found, otherwise None
"""
try:
op = Index.operators[version.op]
except KeyError:
raise ValueError(
"Unsupported operation: {0}"
.format(version.op)
)
return op(versions, version.edge)

View File

@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from collections import namedtuple
from packetary.objects.base import ComparableObject
FileChecksum = namedtuple("FileChecksum", ("md5", "sha1", "sha256"))
class Package(ComparableObject):
"""Structure to describe package object."""
def __init__(self, repository, name, version, filename,
filesize, checksum, mandatory=False,
requires=None, provides=None, obsoletes=None):
"""Initialises.
:param name: the package`s name
:param version: the package`s version
:param filename: the package`s relative filename
:param filesize: the package`s file size
:param checksum: the package`s checksum
:param requires: the package`s requirements(optional)
:param provides: the package`s provides(optional)
:param obsoletes: the package`s obsoletes(optional)
:param mandatory: indicates that package is mandatory
"""
self.repository = repository
self.name = name
self.version = version
self.filename = filename
self.checksum = checksum
self.filesize = filesize
self.requires = requires or []
self.provides = provides or []
self.obsoletes = obsoletes or []
self.mandatory = mandatory
def __copy__(self):
"""Creates shallow copy of package."""
return Package(**self.__dict__)
def __str__(self):
return "{0} {1}".format(self.name, self.version)
def __unicode__(self):
return u"{0} {1}".format(self.name, self.version)
def __hash__(self):
return hash((self.name, self.version))
def cmp(self, other):
"""Compares with other Package object."""
if self.name < other.name:
return -1
if self.name > other.name:
return 1
if self.version < other.version:
return -1
if self.version > other.version:
return 1
return 0

View File

@ -0,0 +1,154 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import operator
class VersionRange(object):
"""Describes the range of versions.
Range of version is compare operation and edge.
the compare operation can be one of:
equal, greater, less, greater or equal, less or equal.
"""
def __init__(self, op=None, edge=None):
"""Initialises.
:param op: the name of operator to compare.
:param edge: the edge of versions.
"""
self.op = op
self.edge = edge
def __hash__(self):
return hash((self.op, self.edge))
def __eq__(self, other):
if not isinstance(other, VersionRange):
return False
return self.op == other.op and \
self.edge == other.edge
def __str__(self):
if self.edge is not None:
return "{0} {1}".format(self.op, self.edge)
return "any"
def __unicode__(self):
if self.edge is not None:
return u"{0} {1}".format(self.op, self.edge)
return u"any"
def has_intersection(self, other):
"""Checks that 2 ranges has intersection."""
if not isinstance(other, VersionRange):
raise TypeError(
"Unorderable type <type 'VersionRange'> and {0}"
.format(type(other))
)
if self.op is None or other.op is None:
return True
my_op = getattr(operator, self.op)
other_op = getattr(operator, other.op)
if self.op[0] == other.op[0]:
if self.op[0] == 'l':
if self.edge < other.edge:
return my_op(self.edge, other.edge)
return other_op(other.edge, self.edge)
elif self.op[0] == 'g':
if self.edge > other.edge:
return my_op(self.edge, other.edge)
return other_op(other.edge, self.edge)
if self.op == 'eq':
return other_op(self.edge, other.edge)
if other.op == 'eq':
return my_op(other.edge, self.edge)
return (
my_op(other.edge, self.edge) and
other_op(self.edge, other.edge)
)
class PackageRelation(object):
"""Describes the package`s relation.
Relation includes the name of required package
and range of versions that satisfies requirement.
"""
def __init__(self, name, version=None, alternative=None):
"""Initialises.
:param name: the name of required package
:param version: the version range of required package
:param alternative: the alternative relation
"""
self.name = name
self.version = VersionRange() if version is None else version
self.alternative = alternative
@classmethod
def from_args(cls, *args):
"""Construct relation from list of arguments.
:param args: the list of tuples(name, [version_op, version_edge])
"""
if len(args) == 0:
return None
head = args[0]
name = head[0]
version = VersionRange(*head[1:])
alternative = cls.from_args(*args[1:])
return cls(name, version, alternative)
def __iter__(self):
"""Iterates over alternatives."""
r = self
while r is not None:
yield r
r = r.alternative
def __hash__(self):
return hash((self.name, self.version))
def __eq__(self, other):
if not isinstance(other, PackageRelation):
return False
return self.name == other.name and \
self.version == other.version
def __str__(self):
if self.alternative is None:
return "{0} ({1})".format(self.name, self.version)
return "{0} ({1}) | {2}".format(
self.name, self.version, self.alternative
)
def __unicode__(self):
if self.alternative is None:
return u"{0} ({1})".format(self.name, self.version)
return u"{0} ({1}) | {2}".format(
self.name, self.version, self.alternative
)

View File

@ -0,0 +1,44 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class Repository(object):
"""Structure to describe repository object."""
def __init__(self, name, url, architecture, origin):
"""Initialises.
:param name: the repository`s name, may be tuple of strings
:param url: the repository`s URL
:param architecture: the repository`s architecture
:param origin: the repository`s origin
"""
self.name = name
self.url = url
self.architecture = architecture
self.origin = origin
def __str__(self):
if isinstance(self.name, tuple):
return ".".join(self.name)
return str(self.name)
def __unicode__(self):
if isinstance(self.name, tuple):
return u".".join(self.name)
return unicode(self.name, "utf8")
def __copy__(self):
"""Creates shallow copy of package."""
return Repository(**self.__dict__)

View File

View File

@ -0,0 +1,50 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from packetary import objects
def gen_repository(name="test", url="file:///test",
architecture="x86_64", origin="Test"):
"""Helper to create Repository object with default attributes."""
return objects.Repository(name, url, architecture, origin)
def gen_relation(name="test", version=None):
"""Helper to create PackageRelation object with default attributes."""
return [
objects.PackageRelation(
name=name, version=objects.VersionRange(*(version or []))
)
]
def gen_package(idx=1, **kwargs):
"""Helper to create Package object with default attributes."""
repository = gen_repository()
kwargs.setdefault("name", "package{0}".format(idx))
kwargs.setdefault("repository", repository)
kwargs.setdefault("version", 1)
kwargs.setdefault("checksum", objects.FileChecksum("1", "2", "3"))
kwargs.setdefault("filename", "test.pkg")
kwargs.setdefault("filesize", 1)
for relation in ("requires", "provides", "obsoletes"):
if relation not in kwargs:
kwargs[relation] = gen_relation(
"{0}{1}".format(relation, idx), ["le", idx + 1]
)
return objects.Package(**kwargs)

View File

@ -22,127 +22,121 @@ from packetary.library import connections
from packetary.tests import base
class TestConnectionsPool(base.TestCase):
def test_get_connection(self):
pool = connections.ConnectionsPool(count=2)
self.assertEqual(2, pool.free.qsize())
with pool.get():
self.assertEqual(1, pool.free.qsize())
self.assertEqual(2, pool.free.qsize())
@mock.patch("packetary.library.connections.logger")
class TestConnectionManager(base.TestCase):
def _check_proxies(self, manager, http_proxy, https_proxy):
for h in manager.opener.handlers:
if isinstance(h, connections.urllib.ProxyHandler):
self.assertEqual(
(http_proxy, https_proxy),
(h.proxies["http"], h.proxies["https"])
)
break
else:
self.fail("ProxyHandler should be in list of handlers.")
def _check_proxies(self, pool, http_proxy, https_proxy):
with pool.get() as c:
for h in c.opener.handlers:
if isinstance(h, connections.urllib_request.ProxyHandler):
self.assertEqual(
(http_proxy, https_proxy),
(h.proxies["http"], h.proxies["https"])
)
break
else:
self.fail("ProxyHandler should be in list of handlers.")
def test_set_proxy(self):
pool = connections.ConnectionsPool(count=1, proxy="http://localhost")
self._check_proxies(pool, "http://localhost", "http://localhost")
pool = connections.ConnectionsPool(
def test_set_proxy(self, _):
manager = connections.ConnectionsManager(proxy="http://localhost")
self._check_proxies(
manager, "http://localhost", "http://localhost"
)
manager = connections.ConnectionsManager(
proxy="http://localhost", secure_proxy="https://localhost")
self._check_proxies(pool, "http://localhost", "https://localhost")
self._check_proxies(
manager, "http://localhost", "https://localhost"
)
manager = connections.ConnectionsManager(retries_num=2)
self.assertEqual(2, manager.retries_num)
for h in manager.opener.handlers:
if isinstance(h, connections.RetryHandler):
break
else:
self.fail("RetryHandler should be in list of handlers.")
def test_reliability(self):
pool = connections.ConnectionsPool(count=0, retries_num=2)
self.assertEqual(1, pool.free.qsize())
with pool.get() as c:
self.assertEqual(2, c.retries_num)
for h in c.opener.handlers:
if isinstance(h, connections.RetryHandler):
break
else:
self.fail("RetryHandler should be in list of handlers.")
class TestConnection(base.TestCase):
def setUp(self):
super(TestConnection, self).setUp()
self.connection = connections.Connection(mock.MagicMock(), 2)
def test_make_request(self):
request = self.connection.make_request("/test/file", 0)
@mock.patch("packetary.library.connections.urllib.build_opener")
def test_make_request(self, *_):
manager = connections.ConnectionsManager(retries_num=2)
request = manager.make_request("/test/file", 0)
self.assertIsInstance(request, connections.RetryableRequest)
self.assertEqual("file:///test/file", request.get_full_url())
self.assertEqual(0, request.offset)
self.assertEqual(2, request.retries_left)
request2 = self.connection.make_request("http://server/path", 100)
request2 = manager.make_request("http://server/path", 100)
self.assertEqual("http://server/path", request2.get_full_url())
self.assertEqual(100, request2.offset)
def test_open_stream(self):
self.connection.open_stream("/test/file")
self.assertEqual(1, self.connection.opener.open.call_count)
args = self.connection.opener.open.call_args[0]
@mock.patch("packetary.library.connections.urllib.build_opener")
def test_open_stream(self, *_):
manager = connections.ConnectionsManager(retries_num=2)
manager.open_stream("/test/file")
self.assertEqual(1, manager.opener.open.call_count)
args = manager.opener.open.call_args[0]
self.assertIsInstance(args[0], connections.RetryableRequest)
self.assertEqual(2, args[0].retries_left)
@mock.patch("packetary.library.connections.logger")
def test_retries_on_io_error(self, logger):
self.connection.opener.open.side_effect = [
@mock.patch("packetary.library.connections.urllib.build_opener")
def test_retries_on_io_error(self, _, logger):
manager = connections.ConnectionsManager(retries_num=2)
manager.opener.open.side_effect = [
IOError("I/O error"),
mock.MagicMock()
]
self.connection.open_stream("/test/file")
self.assertEqual(2, self.connection.opener.open.call_count)
manager.open_stream("/test/file")
self.assertEqual(2, manager.opener.open.call_count)
logger.exception.assert_called_with(
"Failed to open url - %s: %s. retries left - %d.",
"/test/file", "I/O error", 1
)
self.connection.opener.open.side_effect = IOError("I/O error")
manager.opener.open.side_effect = IOError("I/O error")
with self.assertRaises(IOError):
self.connection.open_stream("/test/file")
manager.open_stream("/test/file")
logger.exception.assert_called_with(
"Failed to open url - %s: %s. retries left - %d.",
"/test/file", "I/O error", 0
)
def test_raise_other_errors(self):
self.connection.opener.open.side_effect = \
connections.urllib_error.HTTPError("", 500, "", {}, None)
@mock.patch("packetary.library.connections.urllib.build_opener")
def test_raise_other_errors(self, *_):
manager = connections.ConnectionsManager()
manager.opener.open.side_effect = \
connections.urlerror.HTTPError("", 500, "", {}, None)
with self.assertRaises(connections.urllib_error.URLError):
self.connection.open_stream("/test/file")
with self.assertRaises(connections.urlerror.URLError):
manager.open_stream("/test/file")
self.assertEqual(1, self.connection.opener.open.call_count)
self.assertEqual(1, manager.opener.open.call_count)
@mock.patch("packetary.library.connections.urllib.build_opener")
@mock.patch("packetary.library.connections.os")
def test_retrieve_from_offset(self, os):
def test_retrieve_from_offset(self, os, *_):
manager = connections.ConnectionsManager()
os.path.mkdirs.side_effect = OSError(17, "")
os.open.return_value = 1
response = mock.MagicMock()
self.connection.opener.open.return_value = response
manager.opener.open.return_value = response
response.read.side_effect = [b"test", b""]
self.connection.retrieve("/file/src", "/file/dst", 10)
manager.retrieve("/file/src", "/file/dst", 10)
os.lseek.assert_called_once_with(1, 10, os.SEEK_SET)
os.ftruncate.assert_called_once_with(1, 10)
self.assertEqual(1, os.write.call_count)
os.fsync.assert_called_once_with(1)
os.close.assert_called_once_with(1)
@mock.patch.multiple(
"packetary.library.connections",
logger=mock.DEFAULT,
os=mock.DEFAULT
)
def test_retrieve_from_offset_fail(self, os, logger):
os.path.mkdirs.side_effect = OSError(17, "")
@mock.patch("packetary.library.connections.urllib.build_opener")
@mock.patch("packetary.library.connections.os")
def test_retrieve_from_offset_fail(self, os, _, logger):
manager = connections.ConnectionsManager(retries_num=2)
os.path.mkdirs.side_effect = OSError(connections.errno.EACCES, "")
os.open.return_value = 1
response = mock.MagicMock()
self.connection.opener.open.side_effect = [
manager.opener.open.side_effect = [
connections.RangeError("error"), response
]
response.read.side_effect = [b"test", b""]
self.connection.retrieve("/file/src", "/file/dst", 10)
manager.retrieve("/file/src", "/file/dst", 10)
logger.warning.assert_called_once_with(
"Failed to resume download, starts from begin: %s",
"Failed to resume download, starts from the beginning: %s",
"/file/src"
)
os.lseek.assert_called_once_with(1, 0, os.SEEK_SET)

View File

@ -0,0 +1,189 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from packetary.objects.index import Index
from packetary import objects
from packetary.tests import base
from packetary.tests.stubs.generator import gen_package
from packetary.tests.stubs.generator import gen_relation
class TestIndex(base.TestCase):
def test_add(self):
index = Index()
index.add(gen_package(version=1))
self.assertIn("package1", index.packages)
self.assertIn(1, index.packages["package1"])
self.assertIn("obsoletes1", index.obsoletes)
self.assertIn("provides1", index.provides)
index.add(gen_package(version=2))
self.assertEqual(1, len(index.packages))
self.assertIn(1, index.packages["package1"])
self.assertIn(2, index.packages["package1"])
self.assertEqual(1, len(index.obsoletes))
self.assertEqual(1, len(index.provides))
def test_find(self):
index = Index()
p1 = gen_package(version=1)
p2 = gen_package(version=2)
index.add(p1)
index.add(p2)
self.assertIs(
p1,
index.find("package1", objects.VersionRange("eq", 1))
)
self.assertIs(
p2,
index.find("package1", objects.VersionRange())
)
self.assertIsNone(
index.find("package1", objects.VersionRange("gt", 2))
)
def test_find_all(self):
index = Index()
p11 = gen_package(idx=1, version=1)
p12 = gen_package(idx=1, version=2)
p21 = gen_package(idx=2, version=1)
p22 = gen_package(idx=2, version=2)
index.add(p11)
index.add(p12)
index.add(p21)
index.add(p22)
self.assertItemsEqual(
[p11, p12],
index.find_all("package1", objects.VersionRange())
)
self.assertItemsEqual(
[p21, p22],
index.find_all("package2", objects.VersionRange("le", 2))
)
def test_find_newest_package(self):
index = Index()
p1 = gen_package(idx=1, version=2)
p2 = gen_package(idx=2, version=2)
p2.obsoletes.extend(
gen_relation(p1.name, ["lt", p1.version])
)
index.add(p1)
index.add(p2)
self.assertIs(
p1, index.find(p1.name, objects.VersionRange("eq", p1.version))
)
self.assertIs(
p2, index.find(p1.name, objects.VersionRange("eq", 1))
)
def test_find_top_down(self):
index = Index()
p1 = gen_package(version=1)
p2 = gen_package(version=2)
index.add(p1)
index.add(p2)
self.assertIs(
p2,
index.find("package1", objects.VersionRange("le", 2))
)
self.assertIs(
p1,
index.find("package1", objects.VersionRange("lt", 2))
)
self.assertIsNone(
index.find("package1", objects.VersionRange("lt", 1))
)
def test_find_down_up(self):
index = Index()
p1 = gen_package(version=1)
p2 = gen_package(version=2)
index.add(p1)
index.add(p2)
self.assertIs(
p2,
index.find("package1", objects.VersionRange("ge", 2))
)
self.assertIs(
p2,
index.find("package1", objects.VersionRange("gt", 1))
)
self.assertIsNone(
index.find("package1", objects.VersionRange("gt", 2))
)
def test_find_accurate(self):
index = Index()
p1 = gen_package(version=1)
p2 = gen_package(version=2)
index.add(p1)
index.add(p2)
self.assertIs(
p1,
index.find("package1", objects.VersionRange("eq", 1))
)
self.assertIsNone(
index.find("package1", objects.VersionRange("eq", 3))
)
def test_find_obsolete(self):
index = Index()
p1 = gen_package(version=1)
index.add(p1)
self.assertIs(
p1, index.find("obsoletes1", objects.VersionRange("le", 2))
)
self.assertIsNone(
index.find("obsoletes1", objects.VersionRange("gt", 2))
)
def test_find_provides(self):
index = Index()
p1 = gen_package(version=1)
p2 = gen_package(version=2)
index.add(p1)
index.add(p2)
self.assertIs(
p2, index.find("provides1", objects.VersionRange("ge", 2))
)
self.assertIsNone(
index.find("provides1", objects.VersionRange("lt", 2))
)
def test_len(self):
index = Index()
for i in six.moves.range(3):
index.add(gen_package(idx=i + 1))
self.assertEqual(3, len(index))
for i in six.moves.range(3):
index.add(gen_package(idx=i + 1, version=2))
self.assertEqual(6, len(index))
self.assertEqual(3, len(index.packages))
for i in six.moves.range(3):
index.add(gen_package(idx=i + 1, version=2))
self.assertEqual(6, len(index))
self.assertEqual(3, len(index.packages))

View File

@ -0,0 +1,183 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import six
from packetary.objects import PackageRelation
from packetary.objects import VersionRange
from packetary.tests import base
from packetary.tests.stubs import generator
class TestObjectBase(base.TestCase):
def check_copy(self, origin):
clone = copy.copy(origin)
self.assertIsNot(origin, clone)
self.assertEqual(origin, clone)
origin_name = origin.name
origin.name += "1"
self.assertEqual(
origin_name,
clone.name
)
def check_ordering(self, *args):
for i in six.moves.range(len(args) - 1, 1, -1):
self.assertLess(args[i - 1], args[i])
self.assertGreater(args[i], args[i - 1])
def check_equal(self, o1, o11, o2):
self.assertEqual(o1, o11)
self.assertEqual(o11, o1)
self.assertNotEqual(o1, o2)
self.assertNotEqual(o2, o1)
self.assertNotEqual(o1, None)
def check_hashable(self, o1, o2):
d = dict()
d[o1] = o2
d[o2] = o1
self.assertIs(o2, d[o1])
self.assertIs(o1, d[o2])
class TestPackageObject(TestObjectBase):
def test_copy(self):
self.check_copy(generator.gen_package(name="test1"))
def test_ordering(self):
self.check_ordering([
generator.gen_package(name="test1", version=1),
generator.gen_package(name="test1", version=2),
generator.gen_package(name="test2", version=1),
generator.gen_package(name="test2", version=2)
])
def test_equal(self):
self.check_equal(
generator.gen_package(name="test1", version=1),
generator.gen_package(name="test1", version=1),
generator.gen_package(name="test2", version=1)
)
def test_hashable(self):
self.check_hashable(
generator.gen_package(name="test1", version=1),
generator.gen_package(name="test2", version=1),
)
self.check_hashable(
generator.gen_package(name="test1", version=1),
generator.gen_package(name="test1", version=2),
)
class TestRepositoryObject(base.TestCase):
def test_copy(self):
origin = generator.gen_repository()
clone = copy.copy(origin)
self.assertEqual(clone.name, origin.name)
self.assertEqual(clone.architecture, origin.architecture)
class TestRelationObject(TestObjectBase):
def test_equal(self):
self.check_equal(
generator.gen_relation(name="test1"),
generator.gen_relation(name="test1"),
generator.gen_relation(name="test2")
)
def test_hashable(self):
self.check_hashable(
generator.gen_relation(name="test1")[0],
generator.gen_relation(name="test1", version=["le", 1])[0]
)
def test_from_args(self):
r = PackageRelation.from_args(
("test", "le", 2), ("test2",), ("test3",)
)
self.assertEqual("test", r.name)
self.assertEqual("le", r.version.op)
self.assertEqual(2, r.version.edge)
self.assertEqual("test2", r.alternative.name)
self.assertEqual(VersionRange(), r.alternative.version)
self.assertEqual("test3", r.alternative.alternative.name)
self.assertEqual(VersionRange(), r.alternative.alternative.version)
self.assertIsNone(r.alternative.alternative.alternative)
def test_iter(self):
it = iter(PackageRelation.from_args(
("test", "le", 2), ("test2", "ge", 3))
)
self.assertEqual("test", next(it).name)
self.assertEqual("test2", next(it).name)
with self.assertRaises(StopIteration):
next(it)
class TestVersionRange(TestObjectBase):
def test_equal(self):
self.check_equal(
generator.gen_relation(name="test1"),
generator.gen_relation(name="test1"),
generator.gen_relation(name="test2")
)
def test_hashable(self):
self.check_hashable(
VersionRange(op="le"),
VersionRange(op="le", edge=3)
)
def __check_intersection(self, assertion, cases):
for data in cases:
v1 = VersionRange(*data[0])
v2 = VersionRange(*data[1])
assertion(
v1.has_intersection(v2), msg="%s and %s" % (v1, v2)
)
assertion(
v2.has_intersection(v1), msg="%s and %s" % (v2, v1)
)
def test_have_intersection(self):
cases = [
(("lt", 2), ("gt", 1)),
(("lt", 3), ("lt", 4)),
(("gt", 3), ("gt", 4)),
(("eq", 1), ("eq", 1)),
(("ge", 1), ("le", 1)),
(("eq", 1), ("lt", 2)),
((None, None), ("le", 10)),
]
self.__check_intersection(self.assertTrue, cases)
def test_does_not_have_intersection(self):
cases = [
(("lt", 2), ("gt", 2)),
(("ge", 2), ("lt", 2)),
(("gt", 2), ("le", 2)),
(("gt", 1), ("lt", 1)),
]
self.__check_intersection(self.assertFalse, cases)
def test_intersection_is_typesafe(self):
with self.assertRaises(TypeError):
VersionRange("eq", 1).has_intersection(("eq", 1))

View File

@ -2,6 +2,9 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
Babel>=1.3
eventlet>=0.17
pbr>=1.6
Babel>=1.3
eventlet>=0.15
bintrees>=2.0.2
chardet>=2.3.0
six>=1.5.2