Merge "Remove hard-coding of package/component names"

anvil/packaging/base.py

@@ -30,32 +30,6 @@ from anvil import utils
 
 LOG = logging.getLogger(__name__)
 
-# TODO(harlowja): get rid of static lists in code files for these names
-# which we should be able to take in via configuration or other automatic
-# process
-OPENSTACK_PACKAGES = frozenset([
-    "ceilometer",
-    "cinder",
-    "glance",
-    "heat",
-    "horizon",
-    "keystone",
-    "neutron",
-    "nova",
-    "oslo.config",
-    "oslo.messaging",
-    "python-cinderclient",
-    "python-glanceclient",
-    "python-keystoneclient",
-    "python-neutronclient",
-    "python-novaclient",
-    "python-swiftclient",
-    "python-troveclient",
-    "swift",
-    "trove",
-])
-SKIP_PACKAGE_NAMES = []
-
 
 class InstallHelper(object):
     """Run pre and post install for a single package."""
@@ -218,16 +192,18 @@ class DependencyHandler(object):
         extra_pips = extra_pips or []
         cmdline = [
             self.multipip_executable,
-            "--skip-requirements-regex",
+            "--pip", self.pip_executable,
-            "python.*client",
-            "--pip",
-            self.pip_executable
         ]
         cmdline = cmdline + extra_pips + ["-r"] + requires_files
-        cmdline.extend(["--ignore-package"])
+
-        cmdline.extend(OPENSTACK_PACKAGES)
+        ignore_pip_names = set(self.python_names)
-        cmdline.extend(SKIP_PACKAGE_NAMES)
+        more_ignores = self.distro.get_dependency_config('ignore_pip_names',
-        cmdline.extend(self.python_names)
+                                                         quiet=True)
+        if more_ignores:
+            ignore_pip_names.update([str(n) for n in more_ignores])
+        if ignore_pip_names:
+            cmdline.extend(["--ignore-package"])
+            cmdline.extend(ignore_pip_names)
 
         stdout, stderr = sh.execute(cmdline, check_exit_code=False)
         self.pips_to_install = list(utils.splitlines_not_empty(stdout))

anvil/packaging/yum.py

@@ -61,28 +61,6 @@ class YumInstallHelper(base.InstallHelper):
 class YumDependencyHandler(base.DependencyHandler):
     OPENSTACK_EPOCH = 2
     SPEC_TEMPLATE_DIR = "packaging/specs"
-    # TODO(harlowja): get rid of these static lists/mappings from code and move
-    # them to configuration (or elsewhere).
-    API_NAMES = {
-        "nova": "Compute",
-        "glance": "Image",
-        "keystone": "Identity",
-        "cinder": "Volume",
-        "neutron": "Networking",
-    }
-    SERVER_NAMES = [
-        "ceilometer",
-        "cinder",
-        "glance",
-        "heat",
-        "keystone",
-        "neutron",
-        "nova",
-        "trove",
-    ]
-    TRANSLATION_NAMES = {
-        'horizon': "python-django-horizon",
-    }
     YUM_REPO_DIR = "/etc/yum.repos.d/"
     SRC_REPOS = {
         'anvil': 'anvil-source',
@@ -519,56 +497,65 @@ class YumDependencyHandler(base.DependencyHandler):
         sh.gzip(archive_name)
         sh.unlink(archive_name)
 
-    @staticmethod
+    def _find_template_and_rpm_name(self, instance, build_name):
-    def _is_client(instance_name, egg_name):
+        search_names = [
-        for i in [instance_name, egg_name]:
+            [
-            if i and i.endswith("client"):
+                build_name,
-                return True
+                "%s.spec" % build_name,
-        return False
+            ],
-
+        ]
-    def _get_template_and_rpm_name(self, instance):
-        template_name = None
         try:
             egg_name = instance.egg_info['name']
-            if self._is_client(instance.name, egg_name):
+            if any(s.endswith("client") for s in (instance.name, egg_name, build_name)):
-                rpm_name = egg_name
+                search_names.extend([
-                template_name = "python-commonclient.spec"
+                    [
-            elif instance.name in self.SERVER_NAMES:
+                        egg_name,
-                rpm_name = "openstack-%s" % (egg_name)
+                        "python-commonclient.spec",
-            else:
+                    ],
-                rpm_name = self.TRANSLATION_NAMES.get(instance.name)
+                ])
+            search_names.extend([
+                [
+                    "openstack-%s" % (egg_name),
+                    "openstack-%s.spec" % (egg_name),
+                ],
+                [
+                    egg_name,
+                    "%s.spec" % (egg_name),
+                ],
+            ])
         except AttributeError:
-            rpm_name = instance.name
+            pass
-            template_name = "%s.spec" % rpm_name
+
-        return (rpm_name, template_name)
+        # Return the first that exists (if any from this list)
+        for (rpm_name, template_name) in search_names:
+            spec_filename = sh.joinpths(settings.TEMPLATE_DIR,
+                                        self.SPEC_TEMPLATE_DIR, template_name)
+            if sh.isfile(spec_filename):
+                return (rpm_name, template_name)
+        return (None, None)
 
     def _build_openstack_package(self, instance):
         params = self._package_parameters(instance)
         patches = instance.list_patches("package")
         params['patches'] = [sh.basename(fn) for fn in patches]
-        (rpm_name, template_name) = self._get_template_and_rpm_name(instance)
+
+        build_name = instance.get_option('build_name', default_value=instance.name)
+        (rpm_name, template_name) = self._find_template_and_rpm_name(instance, build_name)
         try:
             egg_name = instance.egg_info['name']
             params["version"] = instance.egg_info["version"]
-            if self._is_client(instance.name, egg_name):
+            if any(s.endswith("client") for s in (instance.name, egg_name, build_name)):
                 client_name = utils.strip_prefix_suffix(egg_name, "python-", "client")
                 if not client_name:
                     msg = "Bad client package name %s" % (egg_name)
                     raise excp.PackageException(msg)
                 params["clientname"] = client_name
-                params["apiname"] = self.API_NAMES.get(client_name,
+                params["apiname"] = instance.get_option('api_name',
-                                                       client_name.title())
+                                                        default_value=client_name.title())
         except AttributeError:
-            spec_filename = None
+            pass
-            if template_name:
+
-                spec_filename = sh.joinpths(settings.TEMPLATE_DIR,
+        if all((rpm_name, template_name)):
-                                            self.SPEC_TEMPLATE_DIR,
-                                            template_name)
-            if not spec_filename or not sh.isfile(spec_filename):
-                rpm_name = None
-        if rpm_name:
-            if not template_name:
-                template_name = "%s.spec" % rpm_name
             spec_filename = self._write_spec_file(instance, rpm_name,
                                                   template_name, params)
             self._build_from_spec(instance, spec_filename, patches)

conf/components/cinder.yaml

@@ -6,4 +6,8 @@ api_host: "$(auto:ip)"
 api_port: 8776
 protocol: http
 
+# Used for associating the client package with a human understandable
+# name in its package description (not a code-name, like cinder).
+api_name: "Volume"
+
 ...

conf/components/glance.yaml

@@ -29,4 +29,8 @@ image_cache_dir: "/usr/share/anvil/glance/images"
 # Used by install section in the specfile (conflicts with the client binary...)
 remove_file: "/bin/rm -rf %{buildroot}/usr/bin/glance"
 
+# Used for associating the client package with a human understandable
+# name in its package description (not a code-name, like glance).
+api_name: "Image"
+
 ...

conf/components/horizon.yaml

@@ -1,4 +1,7 @@
 # Settings for component horizon
 ---
 
+# Instead of naming this components package horizon, it will be named this instead
+build_name: "python-django-horizon"
+
 ...

conf/components/keystone.yaml

@@ -11,6 +11,10 @@ service_host: "$(auto:ip)"
 service_port: 5000
 service_proto: http
 
+# Used for associating the client package with a human understandable
+# name in its package description (not a code-name, like keystone).
+api_name: "Identity"
+
 # Test exclusions...
 #
 # TODO(harlowja) these should probably be bugs...

conf/components/neutron.yaml

@@ -6,6 +6,10 @@ api_host: "$(auto:ip)"
 api_port: 9696
 protocol: http
 
+# Used for associating the client package with a human understandable
+# name in its package description (not a code-name, like neutron).
+api_name: "Networking"
+
 core_plugin: openvswitch
 use_namespaces: True
 

conf/components/nova.yaml

@@ -117,6 +117,10 @@ vncserver_listen: 127.0.0.1
 vncserver_proxyclient_address: ""
 xvpvncproxy_url: "http://$(auto:ip):6081/console"
 
+# Used for associating the client package with a human understandable
+# name in its package description (not a code-name, like nova).
+api_name: "Compute"
+
 # Test exclusions...
 #
 # TODO(harlowja) these should probably be bugs...

conf/distros/rhel.yaml

@@ -24,9 +24,9 @@ dependency_handler:
         qpid-python: python-qpid # Why is this one backwards :-/
         PyYAML: PyYAML
     arch_dependent:
-    -   selenium
+    - selenium
-    -   xattr
+    - xattr
-    -   PuLP
+    - PuLP
 commands:
     service:
         restart: service $NAME restart