Add fuel_package_updates package

New utility packages from a specified repo and
optionally configures it for a specified
Fuel environment.

Added fuel_package_updates to flake8 tests

blueprint package-fuel-components

Change-Id: I2d6d1f0d3823fe53b995d710ebce190ac1dab0b5

fuel_upgrade_system/fuel_package_updates/MANIFEST.in

@@ -0,0 +1 @@
+include *requirements.txt

fuel_upgrade_system/fuel_package_updates/fuel_package_updates/__init__.py

@@ -0,0 +1,13 @@
+#    Copyright 2014 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

fuel_upgrade_system/fuel_package_updates/fuel_package_updates/fuel_package_updates.py

@@ -0,0 +1,507 @@
+#!/usr/bin/python
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from copy import deepcopy
+import functools
+import json
+import logging
+import os
+import re
+import string
+import subprocess
+import sys
+import traceback
+import urllib2
+import yaml
+import zlib
+
+try:
+    from collections import OrderedDict
+except Exception:
+    # python 2.6 or earlier use backport
+    from ordereddict import OrderedDict
+
+from keystoneclient import exceptions
+from keystoneclient.v2_0 import Client as keystoneclient
+
+from optparse import OptionParser
+from urllib2 import urlopen
+from urlparse import urlparse
+from xml.dom.minidom import parseString
+
+logger = logging.getLogger(__name__)
+
+
+KEYSTONE_CREDS = {'username': os.environ.get('KEYSTONE_USERNAME', 'admin'),
+                  'password': os.environ.get('KEYSTONE_PASSWORD', 'admin'),
+                  'tenant_name': os.environ.get('KEYSTONE_TENANT', 'admin')}
+
+
+class Settings(object):
+    supported_distros = ('centos', 'ubuntu',)
+    supported_releases = ('2014.2-6.1', )
+    updates_destinations = {
+        'centos': r'/var/www/nailgun/{0}/centos/updates',
+        'ubuntu': r'/var/www/nailgun/{0}/ubuntu/updates',
+    }
+    exclude_dirs = ('repodata/', 'mos?.?/')
+    httproot = "/var/www/nailgun"
+    port = 8000
+
+
+class HTTPClient(object):
+
+    def __init__(self, url, keystone_url, credentials, **kwargs):
+        logger.debug('Initiate HTTPClient with url %s', url)
+        self.url = url
+        self.keystone_url = keystone_url
+        self.creds = dict(credentials, **kwargs)
+        self.keystone = None
+        self.opener = urllib2.build_opener(urllib2.HTTPHandler)
+
+    def authenticate(self):
+        try:
+            logger.debug('Initialize keystoneclient with url %s',
+                         self.keystone_url)
+            self.keystone = keystoneclient(
+                auth_url=self.keystone_url, **self.creds)
+            # it depends on keystone version, some versions doing auth
+            # explicitly some dont, but we are making it explicitly always
+            self.keystone.authenticate()
+            logger.debug('Authorization token is successfully updated')
+        except exceptions.AuthorizationFailure:
+            logger.warning(
+                'Cant establish connection to keystone with url %s',
+                self.keystone_url)
+
+    @property
+    def token(self):
+        if self.keystone is not None:
+            try:
+                return self.keystone.auth_token
+            except exceptions.AuthorizationFailure:
+                logger.warning(
+                    'Cant establish connection to keystone with url %s',
+                    self.keystone_url)
+            except exceptions.Unauthorized:
+                logger.warning("Keystone returned unauthorized error, trying "
+                               "to pass authentication.")
+                self.authenticate()
+                return self.keystone.auth_token
+        return None
+
+    def get(self, endpoint):
+        req = urllib2.Request(self.url + endpoint)
+        return self._open(req)
+
+    def post(self, endpoint, data=None, content_type="application/json"):
+        if not data:
+            data = {}
+        logger.info('self url is %s' % self.url)
+        req = urllib2.Request(self.url + endpoint, data=json.dumps(data))
+        req.add_header('Content-Type', content_type)
+        return self._open(req)
+
+    def put(self, endpoint, data=None, content_type="application/json"):
+        if not data:
+            data = {}
+        req = urllib2.Request(self.url + endpoint, data=json.dumps(data))
+        req.add_header('Content-Type', content_type)
+        req.get_method = lambda: 'PUT'
+        return self._open(req)
+
+    def delete(self, endpoint):
+        req = urllib2.Request(self.url + endpoint)
+        req.get_method = lambda: 'DELETE'
+        return self._open(req)
+
+    def _open(self, req):
+        try:
+            return self._get_response(req)
+        except urllib2.HTTPError as e:
+            if e.code == 401:
+                logger.warning('Authorization failure: {0}'.format(e.read()))
+                self.authenticate()
+                return self._get_response(req)
+            else:
+                raise
+
+    def _get_response(self, req):
+        if self.token is not None:
+            try:
+                logger.debug('Set X-Auth-Token to {0}'.format(self.token))
+                req.add_header("X-Auth-Token", self.token)
+            except exceptions.AuthorizationFailure:
+                logger.warning('Failed with auth in http _get_response')
+                logger.warning(traceback.format_exc())
+        return self.opener.open(req)
+
+
+def repo_merge(a, b):
+    '''merges two lists of repositories. b replaces records from a.'''
+    if not isinstance(b, list):
+        return deepcopy(b)
+    result = OrderedDict()
+    for repo in a:
+        result[repo['name']] = repo
+    for repo in b:
+        result[repo['name']] = repo
+    return result.values()
+
+
+class FuelWebClient(object):
+
+    def __init__(self, admin_node_ip):
+        self.admin_node_ip = admin_node_ip
+        self.client = NailgunClient(admin_node_ip)
+        super(FuelWebClient, self).__init__()
+
+    def environment(self):
+        """Environment Model
+        :rtype: EnvironmentModel
+        """
+        return self._environment
+
+    def update_cluster_repos(self,
+                             cluster_id,
+                             settings=None):
+        """Updates a cluster with new settings
+        :param cluster_id:
+        :param settings:
+        """
+
+        if settings is None:
+            settings = {}
+
+        attributes = self.client.get_cluster_attributes(cluster_id)
+
+        if 'repo_setup' in attributes['editable']:
+            repos_attr = attributes['editable']['repo_setup']['repos']
+            repos_attr['value'] = repo_merge(repos_attr['value'], settings)
+
+        logger.debug("Try to update cluster "
+                     "with next attributes {0}".format(attributes))
+        self.client.update_cluster_attributes(cluster_id, attributes)
+
+
+class NailgunClient(object):
+    def __init__(self, admin_node_ip, **kwargs):
+        url = "http://{0}:8000".format(admin_node_ip)
+        logger.debug('Initiate Nailgun client with url %s', url)
+        self.keystone_url = "http://{0}:5000/v2.0".format(admin_node_ip)
+        self._client = HTTPClient(url=url, keystone_url=self.keystone_url,
+                                  credentials=KEYSTONE_CREDS,
+                                  **kwargs)
+        super(NailgunClient, self).__init__()
+
+    def json_parse(func):
+        @functools.wraps(func)
+        def wrapped(*args, **kwargs):
+            response = func(*args, **kwargs)
+            return json.loads(response.read())
+        return wrapped
+
+    @property
+    def client(self):
+        return self._client
+
+    @json_parse
+    def get_cluster_attributes(self, cluster_id):
+        return self.client.get(
+            "/api/clusters/{0}/attributes/".format(cluster_id)
+        )
+
+    @json_parse
+    def update_cluster_attributes(self, cluster_id, attrs):
+        return self.client.put(
+            "/api/clusters/{0}/attributes/".format(cluster_id),
+            attrs
+        )
+
+
+class UpdatePackagesException(Exception):
+    pass
+
+
+def exec_cmd(cmd):
+    logger.debug('Execute command "%s"', cmd)
+    child = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        shell=True)
+
+    logger.debug('Stdout and stderr of command "%s":', cmd)
+    for line in child.stdout:
+        logger.debug(line.rstrip())
+
+    return _wait_and_check_exit_code(cmd, child)
+
+
+def _wait_and_check_exit_code(cmd, child):
+    child.wait()
+    exit_code = child.returncode
+    logger.debug('Command "%s" was executed', cmd)
+    return exit_code
+
+
+def get_repository_packages(remote_repo_url, distro):
+    repo_url = urlparse(remote_repo_url)
+    packages = []
+    if distro in ('ubuntu',):
+        packages_url = '{0}/Packages'.format(repo_url.geturl())
+        pkgs_raw = urlopen(packages_url).read()
+        for pkg in pkgs_raw.split('\n'):
+            match = re.search(r'^Package: (\S+)\s*$', pkg)
+            if match:
+                packages.append(match.group(1))
+    elif distro in ('centos',):
+        packages_url = '{0}/repodata/primary.xml.gz'.format(repo_url.geturl())
+        pkgs_xml = parseString(zlib.decompressobj(zlib.MAX_WBITS | 32).
+                               decompress(urlopen(packages_url).read()))
+        for pkg in pkgs_xml.getElementsByTagName('package'):
+            packages.append(
+                pkg.getElementsByTagName('name')[0].firstChild.nodeValue)
+    return packages
+
+
+def get_ubuntu_repos(repopath, ip, httproot, port, baseurl=None):
+    # TODO(mattymo): parse all repo metadata
+    repolist = ['mos6.1-updates', 'mos6.1-security', 'mos6.1-holdback']
+    if baseurl:
+        repourl = "{baseurl}/{repopath}".format(
+            baseurl=baseurl,
+            repopath=repopath.replace(httproot, ''))
+    else:
+        repourl = "http://{ip}:{port}/{repopath}".format(
+            ip=ip,
+            port=port,
+            repopath=repopath.replace(httproot, ''))
+
+    repos = []
+    for repo in repolist:
+        # FIXME(mattymo): repositories cannot have a period in their name
+        name = repo.replace('6.1', '')
+        repoentry = {
+            "type": "deb",
+            "name": name,
+            "uri": repourl,
+            "suite": repo,
+            "section": "main restricted",
+            "priority": 1050}
+        if "holdback" in repo:
+            repoentry['priority'] = 1100
+        repos.append(repoentry)
+    return repos
+
+
+def get_centos_repos(repopath, ip, httproot, port, baseurl=None):
+    if baseurl:
+        repourl = "{baseurl}/{repopath}".format(
+            baseurl=baseurl,
+            repopath=repopath.replace(httproot, ''))
+    else:
+        repourl = "http://{ip}:{port}/{repopath}".format(
+            ip=ip,
+            port=port,
+            repopath=repopath.replace(httproot, ''))
+
+    repoentry = {
+        "type": "rpm",
+        "name": "MOS-Updates",
+        "uri": repourl,
+        "priority": 20}
+    return [repoentry]
+
+
+def reindent(s, numSpaces):
+    s = string.split(s, '\n')
+    s = [(numSpaces * ' ') + line for line in s]
+    s = string.join(s, '\n')
+    return s
+
+
+def show_env_conf(repos, showuri=False, ip="10.20.0.2"):
+    print("Your repositories are now ready for use. You will need to update "
+          "your Fuel environment configuration to use these repositories.")
+    print("Note: Be sure to replace ONLY the repositories listed below.\n")
+    if not showuri:
+        print("Replace the entire repos section of your environment using "
+              "the following commands:\n  fuel --env 1 env --attributes "
+              "--download\n  vim cluster_1/attributes.yaml\n  fuel --env "
+              "1 env --attributes --upload")
+
+    if showuri:
+        for repo in repos:
+            if repo['type'] == "deb":
+                print("{name}:\ndeb {uri} {suite} {section}".format(
+                      name=repo['name'],
+                      uri=repo['uri'],
+                      suite=repo['suite'],
+                      section=repo['section']))
+            else:
+                print("{name}:\n{uri}".format(
+                      name=repo['name'],
+                      uri=repo['uri']))
+    else:
+        spaces = 10
+        yamldata = {"repos": repos}
+        print(reindent(yaml.dump(yamldata, default_flow_style=False), spaces))
+
+
+def update_env_conf(ip, env_id, distro, repos):
+    fwc = FuelWebClient(ip)
+    fwc.update_cluster_repos(env_id, repos)
+
+
+def mirror_remote_repository(remote_repo_url, local_repo_path, exclude_dirs,
+                             distro):
+    repo_url = urlparse(remote_repo_url)
+    cut_dirs = len(repo_url.path.strip('/').split('/'))
+    if "rsync://" in remote_repo_url:
+        excl_dirs = "ubuntu/dists/mos?.?/,repodata/"
+        download_cmd = ('rsync --exclude="*.key","*.gpg",{excl_dirs} -vPr '
+                        '{url} {path}').format(pwd=repo_url.path.rstrip('/'),
+                                               path=local_repo_path,
+                                               excl_dirs=excl_dirs,
+                                               url=repo_url.geturl())
+    else:
+        excl_dirs = "--exclude-directories='ubuntu/dists/mos?.?/,repodata'"
+        download_cmd = (
+            'wget --recursive --no-parent --no-verbose -R "*.html" -R '
+            '"*.gif" -R "*.key" -R "*.gpg" -R "*.dsc" -R "*.tar.gz" '
+            '{excl_dirs} --directory-prefix {path} -nH '
+            '--cut-dirs={cutd} '
+            '{url}').format(pwd=repo_url.path.rstrip('/'),
+                            excl_dirs=excl_dirs,
+                            path=local_repo_path,
+                            cutd=cut_dirs,
+                            url=repo_url.geturl())
+
+    logger.debug('Execute command "%s"', download_cmd)
+    if exec_cmd(download_cmd) != 0:
+        raise UpdatePackagesException('Mirroring of remote packages'
+                                      ' repository failed!')
+
+
+def main():
+    settings = Settings()
+
+    sh = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    sh.setFormatter(formatter)
+    logger.addHandler(sh)
+    logger.setLevel(logging.INFO)
+
+    parser = OptionParser(
+        description="Pull updates for a given release of Fuel based on "
+                    "the provided URL."
+    )
+    parser.add_option('-l', '--list-distros', dest='list_distros',
+                      default=None, action="store_true",
+                      help='List available distributions.')
+    parser.add_option('-d', '--distro', dest='distro', default=None,
+                      help='Distribution name (required)')
+    parser.add_option('-r', '--release', dest='release', default=None,
+                      help='Fuel release name (required)')
+    parser.add_option("-u", "--url", dest="url", default="",
+                      help="Remote repository URL (required)")
+    parser.add_option("-v", "--verbose",
+                      action="store_true", dest="verbose", default=False,
+                      help="Enable debug output")
+    parser.add_option("-i", "--show-uris", dest="showuri", default=False,
+                      action="store_true",
+                      help="Show URIs for new repositories (optional). "
+                      "Useful for WebUI.")
+    parser.add_option("-a", "--apply", dest="apply", default=False,
+                      action="store_true",
+                      help="Apply changes to Fuel environment (optional)")
+    parser.add_option("-e", "--env", dest="env", default=None,
+                      help="Fuel environment ID (required for option -a)")
+    parser.add_option("-s", "--fuel-server", dest="ip", default="10.20.0.2",
+                      help="Address of Fuel Master public address (defaults "
+                      "to 10.20.0.2)")
+    parser.add_option("-b", "--baseurl", dest="baseurl", default=None,
+                      help="URL prefix for mirror, such as http://myserver."
+                      "company.com/repos (optional)")
+    parser.add_option("-p", "--password", dest="admin_pass", default=None,
+                      help="Fuel Master admin password (defaults to admin)."
+                      " Alternatively, use env var KEYSTONE_PASSWORD).")
+
+    (options, args) = parser.parse_args()
+
+    if options.verbose:
+        logger.setLevel(logging.DEBUG)
+
+    if options.list_distros:
+        logger.info("Available distributions:\n  {0}".format(
+            "\n  ".join(settings.supported_distros)))
+        sys.exit(0)
+
+    if options.distro not in settings.supported_distros:
+        raise UpdatePackagesException(
+            'Distro "{0}" is not supported. Please specify one of the '
+            'following: "{1}". See help (--help) for details.'.format(
+                options.distro, ', '.join(settings.supported_distros)))
+
+    if options.release not in settings.supported_releases:
+        raise UpdatePackagesException(
+            'Fuel release "{0}" is not supported. Please specify one of the '
+            'following: "{1}". See help (--help) for details.'.format(
+                options.release, ', '.join(settings.supported_releases)))
+
+    if 'http' not in urlparse(options.url) and 'rsync' not in \
+            urlparse(options.url):
+        raise UpdatePackagesException(
+            'Repository url "{0}" does not look like valid URL. '
+            'See help (--help) for details.'.format(options.url))
+
+    if options.apply and not options.env:
+        raise UpdatePackagesException(
+            '--apply option requires --env to be specified. '
+            'See help (--help) for details.')
+
+    updates_path = settings.updates_destinations[options.distro].format(
+        options.release)
+    if not os.path.exists(updates_path):
+        os.makedirs(updates_path)
+    logger.info('Started mirroring remote repository...')
+    mirror_remote_repository(options.url, updates_path,
+                             settings.exclude_dirs, options.distro)
+    logger.info('Remote repository "{url}" for "{release}" ({distro}) was '
+                'successfuly mirrored to {path} folder.'.format(
+                    url=options.url,
+                    release=options.release,
+                    distro=options.distro,
+                    path=updates_path))
+    if options.distro == "ubuntu":
+        repos = get_ubuntu_repos(updates_path, options.ip, settings.httproot,
+                                 settings.port, options.baseurl)
+    elif options.distro == "centos":
+        repos = get_centos_repos(updates_path, options.ip, settings.httproot,
+                                 settings.port, options.baseurl)
+    else:
+        raise UpdatePackagesException('Unknown distro "{0}"'.format(
+            options.distro))
+    if options.admin_pass:
+        KEYSTONE_CREDS['password'] = options.admin_pass
+    if options.apply:
+        update_env_conf(options.ip, options.env, options.distro, repos)
+    else:
+        show_env_conf(repos, options.showuri, options.ip)
+
+
+if __name__ == '__main__':
+    main()

fuel_upgrade_system/fuel_package_updates/requirements.txt

@@ -0,0 +1,3 @@
+ordereddict>=1.1
+PyYAML==3.10
+python-keystoneclient

fuel_upgrade_system/fuel_package_updates/setup.py

@@ -0,0 +1,49 @@
+#    Copyright 2014 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import os.path
+
+from setuptools import find_packages
+from setuptools import setup
+
+
+def find_requires():
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    requirements = []
+    with open(u'{0}/requirements.txt'.format(dir_path), 'r') as reqs:
+        requirements = reqs.readlines()
+    return requirements
+
+
+if __name__ == "__main__":
+    setup(name='fuel_package_updates',
+          version='6.0.0',
+          description='Package update downloader for Fuel Master node',
+          long_description='Package Update downloader for Fuel Master node',
+          classifiers=[
+              "Programming Language :: Python",
+              "Topic :: System :: Software Distribution"],
+          author='Mirantis Inc.',
+          author_email='product@mirantis.com',
+          url='http://mirantis.com',
+          keywords='fuel update mirantis',
+          packages=find_packages(),
+          zip_safe=False,
+          install_requires=find_requires(),
+          include_package_data=True,
+          entry_points={
+              'console_scripts': [
+                  'fuel-package-updates = fuel_package_updates.fuel_package_'
+                  'updates:main']})

fuel_upgrade_system/fuel_package_updates/test-requirements.txt

@@ -0,0 +1,6 @@
+-r requirements.txt
+hacking==0.10.1
+mock==1.0
+nose==1.1.2
+nose2==0.4.1
+nose-timer==0.2.0

fuel_upgrade_system/fuel_package_updates/tox.ini

@@ -0,0 +1,38 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = py26,py27,pep8
+
+[testenv]
+usedevelop = True
+install_command = pip install --allow-external -U {opts} {packages}
+setenv = VIRTUAL_ENV={envdir}
+deps = -r{toxinidir}/test-requirements.txt
+commands =
+    nosetests {posargs:fuel_package_updates}
+
+[tox:jenkins]
+downloadcache = ~/cache/pip
+
+[testenv:pep8]
+deps = hacking==0.7
+usedevelop = False
+commands =
+    flake8 {posargs:.}
+
+[testenv:venv]
+commands = {posargs:}
+
+[testenv:devenv]
+envdir = devenv
+usedevelop = True
+
+[flake8]
+ignore = H234,H302,H802
+exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs
+show-pep8 = True
+show-source = True
+count = True
+
+[hacking]
+import_exceptions = testtools.matchers

run_tests.sh

@@ -419,6 +419,7 @@ function run_flake8 {
   run_flake8_subproject fuelmenu && \
   run_flake8_subproject network_checker && \
   run_flake8_subproject fuel_upgrade_system/fuel_upgrade && \
+  run_flake8_subproject fuel_upgrade_system/fuel_package_updates && \
   run_flake8_subproject fuel_development && \
   run_flake8_subproject shotgun || result=1
   return $result