Switch to using charm-store for amulet tests

All OpenStack charms are now directly published to the charm store
on landing; switch Amulet helper to resolve charms using the
charm store rather than bzr branches, removing the lag between
charm changes landing and being available for other charms to
use for testing.

This is also important for new layered charms where the charm must
be build and published prior to being consumable.

Change-Id: Iad8888761d1016b833585bd33ca3758e8ade769f

hooks/charmhelpers/contrib/openstack/utils.py

@@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
     related_units,
     relation_ids,
     relation_set,
+    service_name,
     status_set,
     hook_name
 )
@@ -207,6 +208,27 @@ PACKAGE_CODENAMES = {
     ]),
 }
 
+GIT_DEFAULT_REPOS = {
+    'requirements': 'git://github.com/openstack/requirements',
+    'cinder': 'git://github.com/openstack/cinder',
+    'glance': 'git://github.com/openstack/glance',
+    'horizon': 'git://github.com/openstack/horizon',
+    'keystone': 'git://github.com/openstack/keystone',
+    'neutron': 'git://github.com/openstack/neutron',
+    'neutron-fwaas': 'git://github.com/openstack/neutron-fwaas',
+    'neutron-lbaas': 'git://github.com/openstack/neutron-lbaas',
+    'neutron-vpnaas': 'git://github.com/openstack/neutron-vpnaas',
+    'nova': 'git://github.com/openstack/nova',
+}
+
+GIT_DEFAULT_BRANCHES = {
+    'icehouse': 'icehouse-eol',
+    'kilo': 'stable/kilo',
+    'liberty': 'stable/liberty',
+    'mitaka': 'stable/mitaka',
+    'master': 'master',
+}
+
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 
@@ -703,6 +725,53 @@ def git_install_requested():
 requirements_dir = None
 
 
+def git_default_repos(projects):
+    """
+    Returns default repos if a default openstack-origin-git value is specified.
+    """
+    service = service_name()
+
+    for default, branch in GIT_DEFAULT_BRANCHES.iteritems():
+        if projects == default:
+
+            # add the requirements repo first
+            repo = {
+                'name': 'requirements',
+                'repository': GIT_DEFAULT_REPOS['requirements'],
+                'branch': branch,
+            }
+            repos = [repo]
+
+            # neutron and nova charms require some additional repos
+            if service == 'neutron':
+                for svc in ['neutron-fwaas', 'neutron-lbaas', 'neutron-vpnaas']:
+                    repo = {
+                        'name': svc,
+                        'repository': GIT_DEFAULT_REPOS[svc],
+                        'branch': branch,
+                    }
+                    repos.append(repo)
+            elif service == 'nova':
+                repo = {
+                    'name': 'neutron',
+                    'repository': GIT_DEFAULT_REPOS['neutron'],
+                    'branch': branch,
+                }
+                repos.append(repo)
+
+            # finally add the current service's repo
+            repo = {
+                'name': service,
+                'repository': GIT_DEFAULT_REPOS[service],
+                'branch': branch,
+            }
+            repos.append(repo)
+
+            return yaml.dump(dict(repositories=repos))
+
+    return projects
+
+
 def _git_yaml_load(projects_yaml):
     """
     Load the specified yaml into a dictionary.

hooks/charmhelpers/core/host.py

@@ -176,7 +176,7 @@ def init_is_systemd():
 
 
 def adduser(username, password=None, shell='/bin/bash', system_user=False,
-            primary_group=None, secondary_groups=None):
+            primary_group=None, secondary_groups=None, uid=None):
     """Add a user to the system.
 
     Will log but otherwise succeed if the user already exists.
@@ -187,15 +187,21 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False,
     :param bool system_user: Whether to create a login or system user
     :param str primary_group: Primary group for user; defaults to username
     :param list secondary_groups: Optional list of additional groups
+    :param int uid: UID for user being created
 
     :returns: The password database entry struct, as returned by `pwd.getpwnam`
     """
     try:
         user_info = pwd.getpwnam(username)
         log('user {0} already exists!'.format(username))
+        if uid:
+            user_info = pwd.getpwuid(int(uid))
+            log('user with uid {0} already exists!'.format(uid))
     except KeyError:
         log('creating user {0}'.format(username))
         cmd = ['useradd']
+        if uid:
+            cmd.extend(['--uid', str(uid)])
         if system_user or password is None:
             cmd.append('--system')
         else:
@@ -230,14 +236,58 @@ def user_exists(username):
     return user_exists
 
 
-def add_group(group_name, system_group=False):
+def uid_exists(uid):
-    """Add a group to the system"""
+    """Check if a uid exists"""
+    try:
+        pwd.getpwuid(uid)
+        uid_exists = True
+    except KeyError:
+        uid_exists = False
+    return uid_exists
+
+
+def group_exists(groupname):
+    """Check if a group exists"""
+    try:
+        grp.getgrnam(groupname)
+        group_exists = True
+    except KeyError:
+        group_exists = False
+    return group_exists
+
+
+def gid_exists(gid):
+    """Check if a gid exists"""
+    try:
+        grp.getgrgid(gid)
+        gid_exists = True
+    except KeyError:
+        gid_exists = False
+    return gid_exists
+
+
+def add_group(group_name, system_group=False, gid=None):
+    """Add a group to the system
+
+    Will log but otherwise succeed if the group already exists.
+
+    :param str group_name: group to create
+    :param bool system_group: Create system group
+    :param int gid: GID for user being created
+
+    :returns: The password database entry struct, as returned by `grp.getgrnam`
+    """
     try:
         group_info = grp.getgrnam(group_name)
         log('group {0} already exists!'.format(group_name))
+        if gid:
+            group_info = grp.getgrgid(gid)
+            log('group with gid {0} already exists!'.format(gid))
     except KeyError:
         log('creating group {0}'.format(group_name))
         cmd = ['addgroup']
+        if gid:
+            cmd.extend(['--gid', str(gid)])
         if system_group:
             cmd.append('--system')
         else:

hooks/charmhelpers/fetch/__init__.py

@@ -398,16 +398,13 @@ def install_remote(source, *args, **kwargs):
     # We ONLY check for True here because can_handle may return a string
     # explaining why it can't handle a given source.
     handlers = [h for h in plugins() if h.can_handle(source) is True]
-    installed_to = None
     for handler in handlers:
         try:
-            installed_to = handler.install(source, *args, **kwargs)
+            return handler.install(source, *args, **kwargs)
         except UnhandledSource as e:
             log('Install source attempt unsuccessful: {}'.format(e),
                 level='WARNING')
-    if not installed_to:
+    raise UnhandledSource("No handler found for source {}".format(source))
-        raise UnhandledSource("No handler found for source {}".format(source))
-    return installed_to
 
 
 def install_from_config(config_var_name):

hooks/charmhelpers/fetch/bzrurl.py

@@ -42,15 +42,23 @@ class BzrUrlFetchHandler(BaseFetchHandler):
         else:
             return True
 
-    def branch(self, source, dest):
+    def branch(self, source, dest, revno=None):
         if not self.can_handle(source):
             raise UnhandledSource("Cannot handle {}".format(source))
+        cmd_opts = []
+        if revno:
+            cmd_opts += ['-r', str(revno)]
         if os.path.exists(dest):
-            check_call(['bzr', 'pull', '--overwrite', '-d', dest, source])
+            cmd = ['bzr', 'pull']
+            cmd += cmd_opts
+            cmd += ['--overwrite', '-d', dest, source]
         else:
-            check_call(['bzr', 'branch', source, dest])
+            cmd = ['bzr', 'branch']
+            cmd += cmd_opts
+            cmd += [source, dest]
+        check_call(cmd)
 
-    def install(self, source, dest=None):
+    def install(self, source, dest=None, revno=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
         if dest:
@@ -59,10 +67,11 @@ class BzrUrlFetchHandler(BaseFetchHandler):
             dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
                                     branch_name)
 
-        if not os.path.exists(dest_dir):
+        if dest and not os.path.exists(dest):
-            mkdir(dest_dir, perms=0o755)
+            mkdir(dest, perms=0o755)
+
         try:
-            self.branch(source, dest_dir)
+            self.branch(source, dest_dir, revno)
         except OSError as e:
             raise UnhandledSource(e.strerror)
         return dest_dir

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -43,9 +43,6 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         self.openstack = openstack
         self.source = source
         self.stable = stable
-        # Note(coreycb): this needs to be changed when new next branches come
-        # out.
-        self.current_next = "trusty"
 
     def get_logger(self, name="deployment-logger", level=logging.DEBUG):
         """Get a logger object that will log to stdout."""
@@ -72,38 +69,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         self.log.info('OpenStackAmuletDeployment:  determine branch locations')
 
-        # Charms outside the lp:~openstack-charmers namespace
+        # Charms outside the ~openstack-charmers
-        base_charms = ['mysql', 'mongodb', 'nrpe']
+        base_charms = {
-
+            'mysql': ['precise', 'trusty'],
-        # Force these charms to current series even when using an older series.
+            'mongodb': ['precise', 'trusty'],
-        # ie. Use trusty/nrpe even when series is precise, as the P charm
+            'nrpe': ['precise', 'trusty'],
-        # does not possess the necessary external master config and hooks.
+        }
-        force_series_current = ['nrpe']
-
-        if self.series in ['precise', 'trusty']:
-            base_series = self.series
-        else:
-            base_series = self.current_next
 
         for svc in other_services:
-            if svc['name'] in force_series_current:
-                base_series = self.current_next
             # If a location has been explicitly set, use it
             if svc.get('location'):
                 continue
-            if self.stable:
+            if svc['name'] in base_charms:
-                temp = 'lp:charms/{}/{}'
+                # NOTE: not all charms have support for all series we
-                svc['location'] = temp.format(base_series,
+                #       want/need to test against, so fix to most recent
-                                              svc['name'])
+                #       that each base charm supports
+                target_series = self.series
+                if self.series not in base_charms[svc['name']]:
+                    target_series = base_charms[svc['name']][-1]
+                svc['location'] = 'cs:{}/{}'.format(target_series,
+                                                    svc['name'])
+            elif self.stable:
+                svc['location'] = 'cs:{}/{}'.format(self.series,
+                                                    svc['name'])
             else:
-                if svc['name'] in base_charms:
+                svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format(
-                    temp = 'lp:charms/{}/{}'
+                    self.series,
-                    svc['location'] = temp.format(base_series,
+                    svc['name']
-                                                  svc['name'])
+                )
-                else:
-                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
-                    svc['location'] = temp.format(self.current_next,
-                                                  svc['name'])
 
         return other_services