Merge "Copied groups index upon fork of repository"

packetary/controllers/repository.py

@@ -170,7 +170,7 @@ class RepositoryController(object):
             utils.get_path_from_url(urljoin(target.url, dst_path)),
             size=package.filesize
         )
-        if package.filesize < 0:
+        if package.filesize <= 0:
             package.filesize = bytes_copied
         if observer:
             observer(bytes_copied)

packetary/drivers/rpm_driver.py

@@ -20,12 +20,12 @@ import copy
 import multiprocessing
 import os
 import shutil
+import tempfile
 
 import createrepo
 import lxml.etree as etree
 import six
 
-
 from packetary.drivers.base import RepositoryDriverBase
 from packetary.library.checksum import composite as checksum_composite
 from packetary.library.streams import GzipDecompress
@@ -159,35 +159,8 @@ class RpmRepositoryDriver(RepositoryDriverBase):
         self.logger.info("loaded: %d packages from %s.", counter, repository)
 
     def add_packages(self, connection, repository, packages):
-        basepath = utils.get_path_from_url(repository.url)
+        groupstree = self._load_groups(connection, repository)
-        self.logger.info("rebuild repository in %s", basepath)
+        self._rebuild_repository(repository, packages, groupstree)
-        md_config = createrepo.MetaDataConfig()
-        try:
-            md_config.workers = multiprocessing.cpu_count()
-            md_config.directory = str(basepath)
-            md_config.update = os.path.exists(
-                os.path.join(basepath, md_config.finaldir)
-            )
-            mdgen = createrepo.MetaDataGenerator(
-                config_obj=md_config, callback=CreaterepoCallBack(self.logger)
-            )
-            mdgen.doPkgMetadata()
-            mdgen.doRepoMetadata()
-            mdgen.doFinalMove()
-        except createrepo.MDError as e:
-            err_msg = six.text_type(e)
-            self.logger.exception(
-                "failed to create yum repository in %s: %s",
-                basepath,
-                err_msg
-            )
-            shutil.rmtree(
-                os.path.join(md_config.outputdir, md_config.tempdir),
-                ignore_errors=True
-            )
-            raise RuntimeError(
-                "Failed to create yum repository in {0}."
-                .format(err_msg))
 
     def fork_repository(self, connection, repository, destination,
                         source=False, locale=False):
@@ -196,7 +169,8 @@ class RpmRepositoryDriver(RepositoryDriverBase):
         new_repo = copy.copy(repository)
         new_repo.url = utils.normalize_repository_url(destination)
         utils.ensure_dir_exist(destination)
-        self.add_packages(connection, new_repo, set())
+        groupstree = self._load_groups(connection, repository)
+        self._rebuild_repository(new_repo, None, groupstree)
         return new_repo
 
     def create_repository(self, repository_data, arch):
@@ -208,6 +182,7 @@ class RpmRepositoryDriver(RepositoryDriverBase):
             origin=repository_data.get('origin')
         )
         utils.ensure_dir_exist(utils.get_path_from_url(repository.url))
+        self._rebuild_repository(repository, None, None)
         return repository
 
     def load_package_from_file(self, repository, filepath):
@@ -236,6 +211,59 @@ class RpmRepositoryDriver(RepositoryDriverBase):
     def get_relative_path(self, repository, filename):
         return "packages/" + filename
 
+    def _rebuild_repository(self, repository, packages, groupstree=None):
+        basepath = utils.get_path_from_url(repository.url)
+        self.logger.info("rebuild repository in %s", basepath)
+        md_config = createrepo.MetaDataConfig()
+        update = packages is not None and \
+            os.path.exists(os.path.join(basepath, md_config.finaldir))
+
+        groupsfile = None
+        if groupstree is not None:
+            with tempfile.NamedTemporaryFile(delete=False) as tmp:
+                groupstree.write(tmp)
+                groupsfile = tmp.name
+        try:
+            md_config.workers = multiprocessing.cpu_count()
+            md_config.directory = str(basepath)
+            md_config.groupfile = groupsfile
+            md_config.update = update
+            if packages is None:
+                # only generate meta-files, without packages info
+                md_config.excludes = ["*"]
+
+            mdgen = createrepo.MetaDataGenerator(
+                config_obj=md_config, callback=CreaterepoCallBack(self.logger)
+            )
+            mdgen.doPkgMetadata()
+            mdgen.doRepoMetadata()
+            mdgen.doFinalMove()
+        except createrepo.MDError as e:
+            err_msg = six.text_type(e)
+            self.logger.exception(
+                "failed to create yum repository in %s: %s",
+                basepath,
+                err_msg
+            )
+            shutil.rmtree(
+                os.path.join(md_config.outputdir, md_config.tempdir),
+                ignore_errors=True
+            )
+            raise RuntimeError(
+                "Failed to create yum repository in {0}."
+                .format(err_msg))
+        finally:
+            if groupsfile is not None:
+                os.unlink(groupsfile)
+
+    def _load_groups(self, connection, repository):
+        repomd = urljoin(repository.url, "repodata/repomd.xml")
+        self.logger.debug("load repomd: %s", repomd)
+        repomd_tree = etree.parse(connection.open_stream(repomd))
+        return self._load_db(
+            connection, repository.url, repomd_tree, "group_gz", "group"
+        )
+
     def _load_db(self, connection, baseurl, repomd, *aliases):
         """Loads database.
 

packetary/tests/stubs/helpers.py

@@ -64,3 +64,7 @@ def get_compressed(stream):
     with closing(gzip.GzipFile(fileobj=compressed, mode="wb")) as gz:
         gz.write(stream.read())
     return Buffer(compressed)
+
+
+def read_to_buffer(stream):
+    return six.BytesIO(stream.read())

packetary/tests/test_rpm_driver.py

@@ -27,6 +27,7 @@ from packetary.schemas import RPM_REPO_SCHEMA
 from packetary.tests import base
 from packetary.tests.stubs.generator import gen_repository
 from packetary.tests.stubs.helpers import get_compressed
+from packetary.tests.stubs.helpers import read_to_buffer
 
 
 REPOMD = path.join(path.dirname(__file__), "data", "repomd.xml")
@@ -53,6 +54,24 @@ class TestRpmDriver(base.TestCase):
         self.createrepo.reset_mock()
         self.connection = mock.MagicMock()
 
+    def configure_streams(self, groups_gzipped=True):
+        streams = []
+        if groups_gzipped:
+            groups_conv = get_compressed
+            md_file = REPOMD
+        else:
+            groups_conv = read_to_buffer
+            md_file = REPOMD2
+
+        for conv, fname in six.moves.zip(
+            (read_to_buffer, groups_conv, get_compressed),
+            (md_file, GROUPS_DB, PRIMARY_DB)
+        ):
+            with open(fname, "rb") as s:
+                streams.append(conv(s))
+
+        self.connection.open_stream.side_effect = streams
+
     def test_priority_sort(self):
         repos = [
             {"name": "repo0"},
@@ -89,17 +108,8 @@ class TestRpmDriver(base.TestCase):
         self.assertEqual("centos", repo.path)
 
     def test_get_packages(self):
-        streams = []
+        self.configure_streams()
-        for conv, fname in zip(
-                (lambda x: six.BytesIO(x.read()),
-                 get_compressed, get_compressed),
-                (REPOMD, GROUPS_DB, PRIMARY_DB)
-        ):
-            with open(fname, "rb") as s:
-                streams.append(conv(s))
-
         packages = []
-        self.connection.open_stream.side_effect = streams
         self.driver.get_packages(
             self.connection,
             gen_repository("test", url="http://host/centos/os/x86_64/"),
@@ -146,18 +156,8 @@ class TestRpmDriver(base.TestCase):
         self.assertFalse(packages[1].mandatory)
 
     def test_get_packages_if_group_not_gzipped(self):
-        streams = []
+        self.configure_streams(False)
-        for conv, fname in zip(
-                (lambda x: six.BytesIO(x.read()),
-                 lambda x: six.BytesIO(x.read()),
-                 get_compressed),
-                (REPOMD2, GROUPS_DB, PRIMARY_DB)
-        ):
-            with open(fname, "rb") as s:
-                streams.append(conv(s))
-
         packages = []
-        self.connection.open_stream.side_effect = streams
         self.driver.get_packages(
             self.connection,
             gen_repository("test", url="http://host/centos/os/x86_64/"),
@@ -170,46 +170,83 @@ class TestRpmDriver(base.TestCase):
         package = packages[0]
         self.assertTrue(package.mandatory)
 
-    @mock.patch("packetary.drivers.rpm_driver.os.path.exists")
+    @mock.patch("packetary.drivers.rpm_driver.os")
-    @mock.patch("packetary.drivers.rpm_driver.shutil")
+    @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
-    def test_add_packages(self, shutil, path_exists):
+    def test_add_packages_to_existing(self, tmp_mock, os_mock):
-        self.createrepo.MDError = ValueError
+        self.configure_streams()
-        self.createrepo.MetaDataGenerator().doFinalMove.side_effect = [
+        tmp_file = mock.MagicMock()
-            None, self.createrepo.MDError()
+        tmp_file.name = "/tmp/groups.gz"
-        ]
+        tmp_mock.return_value.__enter__.return_value = tmp_file
         repo = gen_repository("test", url="file:///repo/os/x86_64")
-        self.createrepo.MetaDataConfig().outputdir = "/repo/os/x86_64"
+        md_gen = mock.MagicMock()
-        self.createrepo.MetaDataConfig().tempdir = "tmp"
+        self.createrepo.MetaDataGenerator.return_value = md_gen
-        self.createrepo.MetaDataConfig().finaldir = "repodata"
+        md_gen.outputdir = "/repo/os/x86_64"
-        path_exists.side_effect = [True, False]
+        md_gen.tempdir = "tmp"
+        md_gen.finaldir = "repodata"
+        os_mock.path.exists.return_value = True
         self.driver.add_packages(self.connection, repo, set())
         self.assertEqual(
             "/repo/os/x86_64",
             self.createrepo.MetaDataConfig().directory
         )
         self.assertTrue(self.createrepo.MetaDataConfig().update)
-        self.createrepo.MetaDataGenerator()\
+        md_gen.doPkgMetadata.assert_called_once_with()
-            .doPkgMetadata.assert_called_once_with()
+        md_gen.doRepoMetadata.assert_called_once_with()
-        self.createrepo.MetaDataGenerator()\
+        md_gen.doFinalMove.assert_called_once_with()
-            .doRepoMetadata.assert_called_once_with()
-        self.createrepo.MetaDataGenerator()\
-            .doFinalMove.assert_called_once_with()
 
-        with self.assertRaises(RuntimeError):
+        self.assertGreater(tmp_file.write.call_count, 0)
-            self.driver.add_packages(self.connection, repo, set())
+        self.assertEqual(
-
+            tmp_file.name, self.createrepo.MetaDataConfig().groupfile
-        self.assertFalse(self.createrepo.MetaDataConfig().update)
-        shutil.rmtree.assert_called_once_with(
-            "/repo/os/x86_64/tmp", ignore_errors=True
         )
+        os_mock.unlink.assert_called_once_with(tmp_file.name)
 
-    @mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
+    @mock.patch("packetary.drivers.rpm_driver.os")
-    def test_fork_repository(self, ensure_dir_exists_mock):
+    @mock.patch("packetary.drivers.rpm_driver.shutil")
-        repo = gen_repository("os", url="http://localhost/os/x86_64/")
+    @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
-        self.createrepo.MetaDataGenerator().doFinalMove.side_effect = [None]
+    def test_add_packages_clean_metadata_on_error(
+            self, tmp_mock, shutil_mock, os_mock
+    ):
+        self.configure_streams()
+        tmp_file = mock.MagicMock()
+        tmp_file.name = "/tmp/groups.gz"
+        tmp_mock.return_value.__enter__.return_value = tmp_file
+        self.createrepo.MDError = ValueError
+        md_gen = mock.MagicMock()
+        self.createrepo.MetaDataGenerator.return_value = md_gen
+        md_gen.doFinalMove.side_effect = self.createrepo.MDError()
+
+        repo = gen_repository("test", url="file:///repo/os/x86_64")
         self.createrepo.MetaDataConfig().outputdir = "/repo/os/x86_64"
         self.createrepo.MetaDataConfig().tempdir = "tmp"
         self.createrepo.MetaDataConfig().finaldir = "repodata"
+        os_mock.path.exists.return_value = True
+        os_mock.path.join.side_effect = lambda *a: '/'.join(a)
+        with self.assertRaises(RuntimeError):
+            self.driver.add_packages(self.connection, repo, set())
+        shutil_mock.rmtree.assert_called_once_with(
+            "/repo/os/x86_64/tmp", ignore_errors=True
+        )
+        os_mock.unlink.assert_called_once_with(tmp_file.name)
+
+    @mock.patch("packetary.drivers.rpm_driver.os")
+    @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
+    @mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
+    def test_fork_repository(
+            self, ensure_dir_exists_mock, tmp_mock, os_mock
+    ):
+        self.configure_streams()
+        tmp_file = mock.MagicMock()
+        tmp_file.name = "/tmp/groups.gz"
+        tmp_mock.return_value.__enter__.return_value = tmp_file
+        repo = gen_repository("os", url="http://localhost/os/x86_64/")
+        md_gen = mock.MagicMock()
+        self.createrepo.MetaDataGenerator.return_value = md_gen
+        md_gen.doFinalMove.side_effect = [None]
+        md_gen.outputdir = "/repo/os/x86_64"
+        md_gen.tempdir = "tmp"
+        md_gen.finaldir = "repodata"
+        md_config = mock.MagicMock()
+        self.createrepo.MetaDataConfig.return_value = md_config
         new_repo = self.driver.fork_repository(
             self.connection,
             repo,
@@ -219,15 +256,33 @@ class TestRpmDriver(base.TestCase):
         self.assertEqual(repo.name, new_repo.name)
         self.assertEqual(repo.architecture, new_repo.architecture)
         self.assertEqual("file:///repo/os/x86_64/", new_repo.url)
-        self.createrepo.MetaDataGenerator()\
+        md_gen.doFinalMove.assert_called_once_with()
-            .doFinalMove.assert_called_once_with()
+        self.assertGreater(tmp_file.write.call_count, 0)
+        self.assertEqual(["*"], md_config.excludes)
+        self.assertFalse(md_config.update)
+        self.assertEqual(tmp_file.name, md_config.groupfile)
+        os_mock.unlink.assert_called_once_with(tmp_file.name)
 
+    @mock.patch("packetary.drivers.rpm_driver.os")
+    @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile")
     @mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist")
-    def test_create_repository(self, ensure_dir_exists_mock):
+    def test_create_repository(
+            self, ensure_dir_exists_mock, tmp_mock, os_mock
+    ):
         repository_data = {
             "name": "Test", "uri": "file:///repo/os/x86_64", "origin": "Test",
             "path": "centos"
         }
+        self.configure_streams()
+        md_gen = mock.MagicMock()
+        self.createrepo.MetaDataGenerator.return_value = md_gen
+        md_gen.doFinalMove.side_effect = [None]
+        md_gen.outputdir = "/repo/os/x86_64"
+        md_gen.tempdir = "tmp"
+        md_gen.finaldir = "repodata"
+        md_config = mock.MagicMock()
+        self.createrepo.MetaDataConfig.return_value = md_config
+
         repo = self.driver.create_repository(repository_data, "x86_64")
         ensure_dir_exists_mock.assert_called_once_with("/repo/os/x86_64/")
         self.assertEqual(repository_data["name"], repo.name)
@@ -235,6 +290,9 @@ class TestRpmDriver(base.TestCase):
         self.assertEqual(repository_data["uri"] + "/", repo.url)
         self.assertEqual(repository_data["origin"], repo.origin)
         self.assertEqual(repository_data["path"], repo.path)
+        md_gen.doFinalMove.assert_called_once_with()
+        self.assertEqual(["*"], md_config.excludes)
+        self.assertFalse(md_config.update)
 
     @mock.patch("packetary.drivers.rpm_driver.utils")
     def test_load_package_from_file(self, utils):