Browse Source

Modify default repos for fuel and fuelmenu

In fuel (nailgun) and fuelmenu we have hardcoded
default repos which we should be able to change
mostly for test purposes.

Default ISO build flow uses fuel packages rebuild
approach, which modifies ubuntu suit as well, eg:
   sync(mos-master) --> (iso)mos10.0 --> deploy(mos10.0)

To use this feature one must use BUILD_PACKAGES=0,
this flag turns off:
  * fuel packages building and
  * ubuntu suit update.
so We consume the same suit as we use during
debmirroring, eg:
   sync(mos-master) --> iso(mos-master) --> deploy(mos-master)

DocImpact
Change-Id: Ia3cefa7c87e35ecd9244a4026b86e772bf569ca9
Closes-bug: #1556125
Vladimir Kozhukalov 2 years ago
parent
commit
e6b165748a
5 changed files with 241 additions and 2 deletions
  1. 12
    0
      iso/bootstrap_admin_node.sh
  2. 206
    0
      iso/fix_default_repos.py
  3. 3
    0
      iso/ks.template
  4. 18
    1
      iso/module.mk
  5. 2
    1
      specs/fuel-main.spec

+ 12
- 0
iso/bootstrap_admin_node.sh View File

@@ -61,6 +61,7 @@ wget \
61 61
 
62 62
 ASTUTE_YAML='/etc/fuel/astute.yaml'
63 63
 BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
64
+CUSTOM_REPOS="/root/default_deb_repos.yaml"
64 65
 bs_build_log='/var/log/fuel-bootstrap-image-build.log'
65 66
 bs_status=0
66 67
 # Backup network configs to this folder. Folder will be created only if
@@ -328,6 +329,11 @@ if (virt-what | fgrep -q "virtualbox") ; then
328 329
   done
329 330
 fi
330 331
 
332
+# change default repo path in fuel-menu before starting any deployment steps
333
+if [ -f "${CUSTOM_REPOS}" ]; then
334
+  fix_default_repos.py fuelmenu --repositories-file "${CUSTOM_REPOS}" || fail
335
+fi
336
+
331 337
 fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail
332 338
 set +x
333 339
 echo "Done!"
@@ -517,6 +523,12 @@ fi
517 523
 
518 524
 # apply puppet
519 525
 /etc/puppet/modules/fuel/examples/deploy.sh || fail
526
+# Update default repo path
527
+if [ -f "${CUSTOM_REPOS}" ]; then
528
+  fix_default_repos.py fuel \
529
+    --repositories-file "${CUSTOM_REPOS}" \
530
+    --release-version "${OPENSTACK_VERSION}" || fail
531
+fi
520 532
 
521 533
 # Sync time
522 534
 systemctl stop ntpd

+ 206
- 0
iso/fix_default_repos.py View File

@@ -0,0 +1,206 @@
1
+#!/usr/bin/env python
2
+#    Copyright 2016 Mirantis, Inc.
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+import argparse
17
+import os
18
+
19
+import six
20
+import yaml
21
+
22
+from fuelclient import objects
23
+
24
+
25
+FUELMENU_DEFAULT_SETTINGS_PATH = \
26
+    "/usr/lib/python2.7/site-packages/fuelmenu/settings.yaml"
27
+
28
+
29
+def is_subdict(dict1, dict2):
30
+    """Checks that dict1 is subdict of dict2.
31
+
32
+    >>> is_subdict({"a": 1}, {'a': 1, 'b': 1})
33
+    True
34
+
35
+    :param dict1: the candidate
36
+    :param dict2: the super dict
37
+    :return: True if all keys from dict1 are present
38
+             and has same value in dict2 otherwise False
39
+    """
40
+    for k, v in six.iteritems(dict1):
41
+        if k not in dict2 or dict2[k] != v:
42
+            return False
43
+    return True
44
+
45
+
46
+def lists_merge(main, patch, key):
47
+    """Merges the list of dicts with same keys.
48
+
49
+    >>> lists_merge([{"a": 1, "c": 2}], [{"a": 1, "c": 3}], key="a")
50
+    [{'a': 1, 'c': 3}]
51
+
52
+    :param main: the main list
53
+    :type main: list
54
+    :param patch: the list of additional elements
55
+    :type patch: list
56
+    :param key: the key for compare
57
+    """
58
+    main_idx = dict(
59
+        (x[key], i) for i, x in enumerate(main)
60
+    )
61
+
62
+    patch_idx = dict(
63
+        (x[key], i) for i, x in enumerate(patch)
64
+    )
65
+
66
+    for k in sorted(patch_idx):
67
+        if k in main_idx:
68
+            main[main_idx[k]].update(patch[patch_idx[k]])
69
+        else:
70
+            main.append(patch[patch_idx[k]])
71
+    return main
72
+
73
+
74
+def update_release_repos(repositories,
75
+                         release_match,
76
+                         replace_repos=False):
77
+    """Applies repositories for existing default settings.
78
+    :param repositories: the meta information of repositories
79
+    :param release_match: The pattern to check Fuel Release
80
+    """
81
+    releases = six.moves.filter(
82
+        lambda x: is_subdict(release_match, x.data),
83
+        objects.Release.get_all()
84
+    )
85
+    for release in releases:
86
+        modified = _update_repository_settings(
87
+            release.data["attributes_metadata"],
88
+            repositories,
89
+            replace_repos=replace_repos)
90
+        if modified:
91
+            release.data["attributes_metadata"] = modified
92
+            print "Try to update the Release '%s'" % release.data['name']
93
+            release.connection.put_request(
94
+                release.instance_api_path.format(release.id),
95
+                release.data
96
+            )
97
+
98
+
99
+def _update_repository_settings(settings,
100
+                                repositories,
101
+                                replace_repos=False):
102
+    """Updates repository settings.
103
+    :param settings: the target settings
104
+    :param repositories: the meta of repositories
105
+    """
106
+    editable = settings["editable"]
107
+    if 'repo_setup' not in editable:
108
+        return
109
+
110
+    repos_attr = editable["repo_setup"]["repos"]
111
+    if replace_repos:
112
+        repos_attr['value'] = repositories
113
+    else:
114
+        lists_merge(repos_attr['value'], repositories, "name")
115
+
116
+    settings["editable"]["repo_setup"]["repos"] = repos_attr
117
+
118
+    return settings
119
+
120
+
121
+def fix_fuel_repos(address, port, user, password,
122
+                   release_version, release_os, repositories):
123
+    os.environ["SERVER_ADDRESS"] = address
124
+    os.environ["LISTEN_PORT"] = port
125
+    os.environ["KEYSTONE_USER"] = user
126
+    os.environ["KEYSTONE_PASS"] = password
127
+
128
+    release_match = {
129
+        "version": release_version,
130
+        "operating_system": release_os
131
+    }
132
+
133
+    update_release_repos(repositories, release_match)
134
+
135
+
136
+def fix_fuelmenu_repos(repositories, replace_repos=False):
137
+    print "Try to update default fuelmenu settings"
138
+    with open(FUELMENU_DEFAULT_SETTINGS_PATH) as f:
139
+        settings = yaml.safe_load(f)
140
+    if replace_repos:
141
+        settings["BOOTSTRAP"]["repos"] = repositories
142
+    else:
143
+        lists_merge(settings["BOOTSTRAP"]["repos"], repositories, "name")
144
+    with open(FUELMENU_DEFAULT_SETTINGS_PATH, "w") as f:
145
+        f.write(yaml.safe_dump(settings, default_flow_style=False))
146
+
147
+
148
+def main():
149
+    parser = argparse.ArgumentParser()
150
+    subparsers = parser.add_subparsers(
151
+        dest="action", help='actions'
152
+    )
153
+    fuel_parser = subparsers.add_parser(
154
+        'fuel', help='fix fuel repos'
155
+    )
156
+    fuel_parser.add_argument(
157
+        '--release-version', dest='release_version', action='store',
158
+        type=str, help='release version', default='newton-10.0'
159
+    )
160
+    fuel_parser.add_argument(
161
+        '--release-os', dest='release_os', action='store',
162
+        type=str, help='release operating system', default='Ubuntu'
163
+    )
164
+    fuel_parser.add_argument(
165
+        '--repositories-file', dest='repositories_file', action='store',
166
+        type=str, help='file where repositories are defined', required=True
167
+    )
168
+    fuel_parser.add_argument(
169
+        '-a', '--address', dest='address', action='store', type=str,
170
+        help='fuel address', default='127.0.0.1'
171
+    )
172
+    fuel_parser.add_argument(
173
+        '-p', '--port', dest='port', action='store', type=str,
174
+        help='fuel port', default='8000'
175
+    )
176
+    fuel_parser.add_argument(
177
+        '--user', dest='user', action='store', type=str,
178
+        help='fuel user', default='admin'
179
+    )
180
+    fuel_parser.add_argument(
181
+        '--password', dest='password', action='store', type=str,
182
+        help='fuel password', default='admin'
183
+    )
184
+    fuelmenu_parser = subparsers.add_parser(
185
+        'fuelmenu', help='fix fuelmenu repos'
186
+    )
187
+    fuelmenu_parser.add_argument(
188
+        '--repositories-file', dest='repositories_file', action='store',
189
+        type=str, help='file where repositories are defined', required=True
190
+    )
191
+    params, other_params = parser.parse_known_args()
192
+
193
+    with open(params.repositories_file) as f:
194
+        repositories = yaml.safe_load(f)
195
+
196
+    if params.action == 'fuel':
197
+        fix_fuel_repos(params.address, params.port,
198
+                       params.user, params.password,
199
+                       params.release_version, params.release_os,
200
+                       repositories)
201
+    else:
202
+        fix_fuelmenu_repos(repositories)
203
+
204
+
205
+if __name__ == "__main__":
206
+    main()

+ 3
- 0
iso/ks.template View File

@@ -390,6 +390,9 @@ OPENSTACK_VERSION=`rpm2cpio ${SOURCE}/mos-centos/Packages/fuel-openstack-metadat
390 390
 test -e ${SOURCE}/fuel_build_number && cp ${SOURCE}/fuel_build_number /etc/fuel_build_number
391 391
 test -e ${SOURCE}/fuel_build_id && cp ${SOURCE}/fuel_build_id /etc/fuel_build_id
392 392
 
393
+# Copy repos config
394
+test -e ${SOURCE}/default_deb_repos.yaml && cp ${SOURCE}/default_deb_repos.yaml /root/default_deb_repos.yaml
395
+
393 396
 # ----------------------
394 397
 # UNPACKING REPOSITORIES
395 398
 # ----------------------

+ 18
- 1
iso/module.mk View File

@@ -31,6 +31,24 @@ $(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/fuel_build_id
31 31
 $(ISOROOT)/fuel_build_id:
32 32
 	echo "$(BUILD_ID)" > $@
33 33
 
34
+##############
35
+# CUSTOM REPOS
36
+##############
37
+
38
+define default_deb_repos
39
+- name: mos
40
+  suite: $(MIRROR_MOS_UBUNTU_SUITE)
41
+endef
42
+
43
+# if we are not building packages and sync repos only, we MUST use
44
+# the same suit as we use during debmirroring
45
+ifeq ($(BUILD_PACKAGES),0)
46
+$(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/default_deb_repos.yaml
47
+endif
48
+$(ISOROOT)/default_deb_repos.yaml: export default_deb_repos_content:=$(default_deb_repos)
49
+$(ISOROOT)/default_deb_repos.yaml:
50
+	/bin/echo -e "$${default_deb_repos_content}\n" > $@
51
+
34 52
 ###############
35 53
 # CENTOS MIRROR
36 54
 ###############
@@ -88,7 +106,6 @@ $(ISOROOT)/ks.cfg: $(SOURCE_DIR)/iso/ks.template $(SOURCE_DIR)/iso/ks.py $(ISORO
88 106
 	python $(SOURCE_DIR)/iso/ks.py \
89 107
 		-t $(SOURCE_DIR)/iso/ks.template \
90 108
 		-c $(ISOROOT)/ks.yaml \
91
-		-u '{"CENTOS_RELEASE": "$(CENTOS_RELEASE)", "PRODUCT_VERSION": "$(PRODUCT_VERSION)"}' \
92 109
 		-o $@.tmp
93 110
 	mv $@.tmp $@
94 111
 

+ 2
- 1
specs/fuel-main.spec View File

@@ -48,6 +48,7 @@ for file in %{_builddir}/%{name}-%{version}/fuel-release/*.repo ; do
48 48
     install -D -m 644 "$file" %{buildroot}/etc/yum.repos.d
49 49
 done
50 50
 install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/bootstrap_admin_node.sh %{buildroot}%{_sbindir}/bootstrap_admin_node.sh
51
+install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/fix_default_repos.py %{buildroot}%{_sbindir}/fix_default_repos.py
51 52
 
52 53
 %clean
53 54
 rm -rf %{buildroot}
@@ -92,4 +93,4 @@ This packages provides script to deploy Fuel components.
92 93
 %files -n fuel-setup
93 94
 %defattr(-,root,root)
94 95
 %{_sbindir}/bootstrap_admin_node.sh
95
-
96
+%{_sbindir}/fix_default_repos.py

Loading…
Cancel
Save