Load pipeline config files from /etc/ceilometer/ firstly

We have moved the pipeline config files from etc/ceilometer/ to
ceilometer/pipeline/data/, but ceilometer don't support to load pipeline
config file from /etc/ceilometer/ and only support to load these config
files from the source code path. This is not user-friendly.

Change-Id: Iedebf6b05e123c94eb7bd200f64beb09382c3969
Closes-Bug: #1670238

ceilometer/event/converter.py

@@ -13,11 +13,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import os
 from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import fnmatch
 from oslo_utils import timeutils
+import pkg_resources
 import six
 
 from ceilometer import declarative
@@ -26,10 +26,7 @@ from ceilometer.i18n import _
 
 OPTS = [
     cfg.StrOpt('definitions_cfg_file',
-               default=os.path.abspath(
+               default="event_definitions.yaml",
-                   os.path.join(
-                       os.path.split(os.path.dirname(__file__))[0],
-                       "pipeline", "data", "event_definitions.yaml")),
                help="Configuration file for event definitions."
                ),
     cfg.BoolOpt('drop_unmatched_notifications',
@@ -298,6 +295,8 @@ def setup_events(conf, trait_plugin_mgr):
     """Setup the event definitions from yaml config file."""
     return NotificationEventsConverter(
         conf,
-        declarative.load_definitions(conf, [],
+        declarative.load_definitions(
-                                     conf.event.definitions_cfg_file),
+            conf, [], conf.event.definitions_cfg_file,
+            pkg_resources.resource_filename(
+                'ceilometer', "pipeline/data/event_definitions.yaml")),
         trait_plugin_mgr)

ceilometer/pipeline.py

@@ -19,6 +19,7 @@ import hashlib
 from itertools import chain
 from operator import methodcaller
 import os
+import pkg_resources
 
 from oslo_config import cfg
 from oslo_log import log
@@ -37,17 +38,11 @@ from ceilometer import sample as sample_util
 
 OPTS = [
     cfg.StrOpt('pipeline_cfg_file',
-               default=os.path.abspath(
+               default="pipeline.yaml",
-                   os.path.join(
-                       os.path.dirname(__file__),
-                       "pipeline", "data", "pipeline.yaml")),
                help="Configuration file for pipeline definition."
                ),
     cfg.StrOpt('event_pipeline_cfg_file',
-               default=os.path.abspath(
+               default="event_pipeline.yaml",
-                   os.path.join(
-                       os.path.dirname(__file__),
-                       "pipeline", "data", "event_pipeline.yaml")),
                help="Configuration file for event pipeline definition."
                ),
     cfg.BoolOpt('refresh_pipeline_cfg',
@@ -667,12 +662,17 @@ class ConfigManagerBase(object):
         self.conf = conf
         self.cfg_loc = None
 
-    def load_config(self, cfg_file):
+    def load_config(self, cfg_file, fallback_cfg_prefix='pipeline/data/'):
         """Load a configuration file and set its refresh values."""
         if os.path.exists(cfg_file):
             self.cfg_loc = cfg_file
         else:
             self.cfg_loc = self.conf.find_file(cfg_file)
+        if not self.cfg_loc and fallback_cfg_prefix is not None:
+            LOG.debug("No pipeline definitions configuration file found! "
+                      "Using default config.")
+            self.cfg_loc = pkg_resources.resource_filename(
+                __name__, fallback_cfg_prefix + cfg_file)
         with open(self.cfg_loc) as fap:
             data = fap.read()
         conf = yaml.safe_load(data)

ceilometer/tests/unit/event/test_converter.py

@@ -770,7 +770,8 @@ class TestNotificationConverter(ConverterBase):
         self.assertTrue(self._convert_message(c, 'info').raw)
         self.assertFalse(self._convert_message(c, 'error').raw)
 
-    def test_setup_events_default_config(self):
+    @mock.patch('ceilometer.declarative.LOG')
+    def test_setup_events_load_config_in_code_tree(self, mocked_log):
         self.CONF.set_override('definitions_cfg_file',
                                '/not/existing/file', group='event')
         self.CONF.set_override('drop_unmatched_notifications',
@@ -778,12 +779,9 @@ class TestNotificationConverter(ConverterBase):
 
         c = converter.setup_events(self.CONF, self.fake_plugin_mgr)
         self.assertIsInstance(c, converter.NotificationEventsConverter)
-        self.assertEqual(1, len(c.definitions))
+        log_called_args = mocked_log.debug.call_args_list
-        self.assertTrue(c.definitions[0].is_catchall)
+        self.assertEqual(
-
+            'No Definitions configuration file found! Using default config.',
-        self.CONF.set_override('drop_unmatched_notifications',
+            log_called_args[0][0][0])
-                               True, group='event')
+        self.assertTrue(log_called_args[1][0][0].startswith(
-
+            'Loading definitions configuration file:'))
-        c = converter.setup_events(self.CONF, self.fake_plugin_mgr)
-        self.assertIsInstance(c, converter.NotificationEventsConverter)
-        self.assertEqual(0, len(c.definitions))

devstack/plugin.sh

@@ -261,13 +261,13 @@ function _ceilometer_configure_storage_backend {
     fi
 
     if [ "$CEILOMETER_BACKEND" = 'mysql' ] || [ "$CEILOMETER_BACKEND" = 'postgresql' ] || [ "$CEILOMETER_BACKEND" = 'mongodb' ]; then
-        sed -i 's/gnocchi:\/\//database:\/\//g' $CEILOMETER_DIR/ceilometer/pipeline/data/event_pipeline.yaml $CEILOMETER_DIR/ceilometer/pipeline/data/pipeline.yaml
+        sed -i 's/gnocchi:\/\//database:\/\//g' $CEILOMETER_CONF_DIR/event_pipeline.yaml $CEILOMETER_CONF_DIR/pipeline.yaml
     fi
 
     # configure panko
     if is_service_enabled panko-api; then
         if ! grep -q 'panko' $CEILOMETER_CONF_DIR/event_pipeline.yaml ; then
-            echo '          - panko://' >> $CEILOMETER_DIR/ceilometer/pipeline/data/event_pipeline.yaml
+            echo '          - panko://' >> $CEILOMETER_CONF_DIR/event_pipeline.yaml
         fi
     fi
 
@@ -304,6 +304,8 @@ function configure_ceilometer {
         cp $CEILOMETER_DIR/etc/ceilometer/$conffile $CEILOMETER_CONF_DIR
     done
 
+    cp $CEILOMETER_DIR/ceilometer/pipeline/data/*.yaml $CEILOMETER_CONF_DIR
+
     if [ "$CEILOMETER_PIPELINE_INTERVAL" ]; then
         sed -i "s/interval:.*/interval: ${CEILOMETER_PIPELINE_INTERVAL}/" $CEILOMETER_CONF_DIR/polling.yaml
     fi

doc/source/install/manual.rst

@@ -125,6 +125,7 @@ Installing the notification agent
 
    $ mkdir -p /etc/ceilometer
    $ cp etc/ceilometer/ceilometer.conf /etc/ceilometer
+   $ cp ceilometer/pipeline/data/*.yaml /etc/ceilometer
 
 5. Edit ``/etc/ceilometer/ceilometer.conf``
 
@@ -202,6 +203,7 @@ Installing the Polling Agent
 
    $ mkdir -p /etc/ceilometer
    $ cp etc/ceilometer/ceilometer.conf /etc/ceilometer/ceilometer.conf
+   $ cp ceilometer/pipeline/data/*.yaml /etc/ceilometer
 
 5. Configure messaging by editing ``/etc/ceilometer/ceilometer.conf``::