Add replace option to apply

The replace option helps to replace existing
repositories with breaking parameters like
priority with generated local repos.

Change-Id: Ib4dc46f8f880478ea555536e7856dab73e2b47a0
Closes-bug: #1540384

.gitignore

@@ -52,3 +52,6 @@ ChangeLog
 *~
 .*.swp
 .*sw?
+
+# Cached filess
+.cache/

contrib/fuel_mirror/data/ubuntu.yaml

@@ -6,8 +6,12 @@ fuel_release_match:
    version: $openstack_version
    operating_system: Ubuntu
 
+# Main is a required parameter which defines what repository will be used
+# for images creation and that mirror should contain all packages for minimal
+# system creation.
 repos:
     -   &ubuntu
+        main: true
         name: "ubuntu"
         uri: *ubuntu_baseurl
         suite: "trusty"
@@ -63,7 +67,8 @@ repos:
         type: "deb"
         priority: 1000
 
-
+# Packages are required to build bootstrap images for a system.
+# The mirror should contiain such packages in addition to local mirror.
 packages: &packages
      - "acpi-support"
      - "anacron"

contrib/fuel_mirror/fuel_mirror/__init__.py

@@ -18,6 +18,9 @@
 
 import pbr.version
 
-
-__version__ = pbr.version.VersionInfo(
-    'fuel_mirror').version_string()
+try:
+    __version__ = pbr.version.VersionInfo(
+        'fuel_mirror').version_string()
+except Exception as e:
+    print("ERROR", e)
+    __version__ = "0.0.0-test"

contrib/fuel_mirror/fuel_mirror/commands/apply.py

@@ -37,6 +37,13 @@ class ApplyCommand(BaseCommand):
             default=False,
             help="Set as default repository."
         )
+        parser.add_argument(
+            "--replace",
+            dest="replace",
+            action="store_true",
+            default=False,
+            help="Replace default repos with generated mirrors."
+        )
         parser.add_argument(
             "-e", "--env",
             dest="env", nargs="+",
@@ -52,6 +59,9 @@ class ApplyCommand(BaseCommand):
 
         data = self.load_data(parsed_args)
         base_url = self.app.config["base_url"]
+        release_match = data["fuel_release_match"]
+        replace_repos = parsed_args.replace
+
         localized_repos = []
         for _, repos in self.get_groups(parsed_args, data):
             for repo_data in repos:
@@ -61,16 +71,29 @@ class ApplyCommand(BaseCommand):
                 )
                 localized_repos.append(new_data)
 
-        release_match = data["fuel_release_match"]
-        self.update_clusters(parsed_args.env, localized_repos, release_match)
+        localized_repos.sort(key=lambda x: not x.pop('main', False))
+
+        self.update_clusters(
+            parsed_args.env,
+            localized_repos,
+            release_match,
+            replace_repos=replace_repos)
+
         if parsed_args.set_default:
-            self.update_default_repos(localized_repos, release_match)
+            self.update_release_repos(
+                localized_repos,
+                release_match,
+                replace_repos=replace_repos)
 
         self.app.stdout.write(
             "Operations have been completed successfully.\n"
         )
 
-    def update_clusters(self, ids, repositories, release_match):
+    def update_clusters(self,
+                        ids,
+                        repositories,
+                        release_match,
+                        replace_repos=False):
         """Applies repositories for existing clusters.
 
         :param ids: the cluster ids.
@@ -94,8 +117,9 @@ class ApplyCommand(BaseCommand):
 
             modified = self._update_repository_settings(
                 cluster.get_settings_data(),
-                repositories
-            )
+                repositories,
+                replace_repos=replace_repos)
+
             if modified:
                 self.app.LOG.info(
                     "Try to update the Cluster '%s'",
@@ -107,21 +131,25 @@ class ApplyCommand(BaseCommand):
                 )
                 cluster.set_settings_data(modified)
 
-    def update_default_repos(self, repositories, release_match):
+    def update_release_repos(self,
+                             repositories,
+                             release_match,
+                             replace_repos=False):
         """Applies repositories for existing default settings.
 
         :param repositories: the meta information of repositories
         :param release_match: The pattern to check Fuel Release
         """
-        self.app.stdout.write("Updating the default repositories...\n")
+        self.app.stdout.write("Updating the release repositories...\n")
         releases = six.moves.filter(
             lambda x: is_subdict(release_match, x.data),
             self.app.fuel.Release.get_all()
         )
         for release in releases:
             if self._update_repository_settings(
-                release.data["attributes_metadata"], repositories
-            ):
+                    release.data["attributes_metadata"],
+                    repositories,
+                    replace_repos=replace_repos):
                 self.app.LOG.info(
                     "Try to update the Release '%s'",
                     release.data['name']
@@ -136,7 +164,10 @@ class ApplyCommand(BaseCommand):
                     release.data
                 )
 
-    def _update_repository_settings(self, settings, repositories):
+    def _update_repository_settings(self,
+                                    settings,
+                                    repositories,
+                                    replace_repos=False):
         """Updates repository settings.
 
         :param settings: the target settings
@@ -144,11 +175,15 @@ class ApplyCommand(BaseCommand):
         """
         editable = settings["editable"]
         if 'repo_setup' not in editable:
-            self.app.LOG.info('Attributes is read-only.')
+            self.app.LOG.info('Attributes are read-only.')
             return
 
         repos_attr = editable["repo_setup"]["repos"]
-        lists_merge(repos_attr['value'], repositories, "name")
+        if replace_repos:
+            repos_attr = {'value': repositories}
+        else:
+            lists_merge(repos_attr['value'], repositories, "name")
+
         return {"editable": {"repo_setup": {"repos": repos_attr}}}
 
 

contrib/fuel_mirror/fuel_mirror/commands/base.py

@@ -17,14 +17,13 @@
 #    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os.path
-from string import Template
 
 from cliff import command
 from jsonschema import validate
 from jsonschema import ValidationError
 import six
-import yaml
 
+from fuel_mirror.common.utils import load_input_data
 from fuel_mirror.schemas.input_data_schema import SCHEMA
 
 
@@ -104,13 +103,13 @@ class BaseCommand(command.Command):
         else:
             input_file = parsed_args.input_file
 
-        with open(input_file, "r") as fd:
-            data = yaml.load(Template(fd.read()).safe_substitute(
-                mos_version=self.app.config["mos_version"],
-                openstack_version=self.app.config["openstack_version"],
-            ))
-            self.validate_data(data, SCHEMA)
-            return data
+        data = load_input_data(
+            input_file,
+            mos_version=self.app.config["mos_version"],
+            openstack_version=self.app.config["openstack_version"]
+        )
+        self.validate_data(data, SCHEMA)
+        return data
 
     @classmethod
     def get_groups(cls, parsed_args, data):

contrib/fuel_mirror/fuel_mirror/common/utils.py

@@ -15,6 +15,7 @@
 #    You should have received a copy of the GNU General Public License along
 #    with this program; if not, write to the Free Software Foundation, Inc.,
 #    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+from string import Template
 
 import six
 import yaml
@@ -92,3 +93,14 @@ def get_fuel_settings():
         }
     except (OSError, IOError):
         return {}
+
+
+def load_input_data(input_file, **kwargs):
+    """Load yaml file and parse it to dict with replacement by kwargs.
+
+    :param input_file: name of file to parse fuel mirror template
+    :param kwargs: arguments to substitute template
+    :return: processed from yaml file dict.
+    """
+    with open(input_file, "r") as fd:
+        return yaml.load(Template(fd.read()).safe_substitute(**kwargs))

contrib/fuel_mirror/fuel_mirror/tests/data/test_ubuntu.yaml

@@ -16,6 +16,7 @@ groups:
     ubuntu:
         -   name: "ubuntu"
             type: "deb"
+            main: true
             uri: "http://localhost/ubuntu"
             suite: "trusty"
             section: "main multiverse restricted universe"

contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py

@@ -29,6 +29,7 @@ subprocess.mswindows = False
 
 from fuel_mirror.commands import apply
 from fuel_mirror.commands import create
+from fuel_mirror.common.utils import load_input_data
 from fuel_mirror.tests import base
 
 
@@ -49,6 +50,25 @@ INVALID_DATA_PATH = os.path.join(
 )
 
 
+# TODO(akostrikov) lists_merge we are using is not stable so we have to use
+# different local repos in cases with merge and without it.
+# We pass sorted by priority list, but in lists_merge we sort it by key.
+# As we are aiming to use existing repos as primary source - it is not issue.
+def local_repos(mirror_host='10.25.0.10:8080', name_postfix='', reverse=True):
+    mirror_lists = load_input_data(UBUNTU_PATH, mos_version=1)
+    sorted_repos = reduce(lambda x, y: x + y, mirror_lists['groups'].values())
+    sorted_repos.sort(key=lambda x: x['priority'], reverse=reverse)
+    for repo in sorted_repos:
+        repo.pop('main', None)
+        repo['name'] = repo['name'] + name_postfix
+        repo['uri'] = repo['uri'].replace('localhost', mirror_host)
+    return sorted_repos
+
+
+def mirror_repos():
+    return local_repos(mirror_host='mirror.com:8080', name_postfix='-mirror')
+
+
 @mock.patch.multiple(
     "fuel_mirror.app",
     accessors=mock.DEFAULT
@@ -72,12 +92,14 @@ class TestCliCommands(base.TestCase):
             "openstack_version": "2"
         }
 
-    def _create_fuel_release(self, fuel_mock, osname):
+    def _create_fuel_release(self, fuel_mock, osname, repos=None):
+        if repos is None:
+            repos = []
         release = mock.MagicMock(data={
             "name": "test release",
             "operating_system": osname,
             "attributes_metadata": {
-                "editable": {"repo_setup": {"repos": {"value": []}}}
+                "editable": {"repo_setup": {"repos": {"value": repos}}}
             }
         })
 
@@ -85,13 +107,15 @@ class TestCliCommands(base.TestCase):
         fuel_mock.Release.get_all.return_value = [release]
         return release
 
-    def _create_fuel_env(self, fuel_mock):
+    def _create_fuel_env(self, fuel_mock, repos=None):
+        if repos is None:
+            repos = []
         env = mock.MagicMock(data={
             "name": "test",
             "release_id": 1
         })
         env.get_settings_data.return_value = {
-            "editable": {"repo_setup": {"repos": {"value": []}}}
+            "editable": {"repo_setup": {"repos": {"value": repos}}}
         }
         fuel_mock.Environment.get_by_ids.return_value = [env]
         fuel_mock.Environment.get_all.return_value = [env]
@@ -194,24 +218,59 @@ class TestCliCommands(base.TestCase):
         )
         fuel.FuelVersion.get_all_data.assert_called_once_with()
         env.set_settings_data.assert_called_with(
-            {'editable': {'repo_setup': {'repos': {'value': [
-                {
-                    'priority': 1000,
-                    'name': 'mos',
-                    'suite': 'mos1',
-                    'section': 'main restricted',
-                    'type': 'deb',
-                    'uri': 'http://10.25.0.10:8080/mos'
-                },
-                {
-                    'priority': 500,
-                    'name': 'ubuntu',
-                    'suite': 'trusty',
-                    'section': 'main multiverse restricted universe',
-                    'type': 'deb',
-                    'uri': 'http://10.25.0.10:8080/ubuntu'
+            {
+                'editable': {
+                    'repo_setup': {
+                        'repos': {'value': local_repos()}
+                    }
                 }
-            ]}}}}
+            }
+        )
+
+    def test_with_existing_mirrors(self, accessors):
+        fuel = accessors.get_fuel_api_accessor()
+        self._setup_fuel_versions(fuel)
+        env = self._create_fuel_env(fuel, repos=mirror_repos())
+        self._create_fuel_release(fuel, "Ubuntu", repos=mirror_repos())
+        self.start_cmd(
+            apply, ['--group', 'mos', 'ubuntu', '--env', '1'],
+            UBUNTU_PATH
+        )
+        accessors.get_fuel_api_accessor.assert_called_with(
+            "10.25.0.10", "test", "test1"
+        )
+        fuel.FuelVersion.get_all_data.assert_called_once_with()
+        env.set_settings_data.assert_called_with(
+            {
+                'editable': {
+                    'repo_setup': {
+                        'repos': {'value': mirror_repos() + local_repos()}
+                    }
+                }
+            }
+        )
+
+    def test_replace_existing_mirrors_with_local(self, accessors):
+        fuel = accessors.get_fuel_api_accessor()
+        self._setup_fuel_versions(fuel)
+        env = self._create_fuel_env(fuel, repos=mirror_repos())
+        self._create_fuel_release(fuel, "Ubuntu", repos=mirror_repos())
+        self.start_cmd(
+            apply, ['--group', 'mos', 'ubuntu', '--env', '1', '--replace'],
+            UBUNTU_PATH
+        )
+        accessors.get_fuel_api_accessor.assert_called_with(
+            "10.25.0.10", "test", "test1"
+        )
+        fuel.FuelVersion.get_all_data.assert_called_once_with()
+        env.set_settings_data.assert_called_with(
+            {
+                'editable': {
+                    'repo_setup': {
+                        'repos': {'value': local_repos(reverse=False)}
+                    }
+                }
+            }
         )
 
     def test_apply_for_centos_based_env(self, accessors):

contrib/fuel_mirror/fuel_mirror/tests/test_utils.py

@@ -102,3 +102,13 @@ class TestUtils(base.TestCase):
             {},
             utils.get_fuel_settings()
         )
+
+    @mock.patch("fuel_mirror.common.utils.yaml")
+    @mock.patch("fuel_mirror.common.utils.open")
+    def test_load_input_data(self, open_mock, yaml_mock):
+        data = "$param1: $param2"
+        open_mock().__enter__().read.return_value = data
+        v = utils.load_input_data("data.yaml", param1="key", param2="value")
+        open_mock.assert_called_with("data.yaml", "r")
+        yaml_mock.load.assert_called_once_with("key: value")
+        self.assertIs(yaml_mock.load(), v)