sack-changer cli
support ability to change sacks via command line. will set new sack value and clear any old sacks (if applicable). backlog must be empty to change value. Change-Id: Icf74b081e4cfaaaa607a5b9c684cbad4b8ecc006
This commit is contained in:
parent
1bacea9416
commit
5b83fe64cc
@ -162,6 +162,39 @@ If the estimated number of metrics is the absolute maximum, divide the value
|
||||
by 500 instead. If the estimated number of active metrics is conservative and
|
||||
expected to grow, divide the value by 100 instead to accommodate growth.
|
||||
|
||||
How do we change sack size
|
||||
--------------------------
|
||||
|
||||
In the event your system grows to capture signficantly more metrics than
|
||||
originally anticipated, the number of sacks can be changed to maintain good
|
||||
distribution. To avoid any loss of data when modifying `sacks` option. The
|
||||
option should be changed in the following order::
|
||||
|
||||
1. Stop all input services (api, statsd)
|
||||
|
||||
2. Stop all metricd services once backlog is cleared
|
||||
|
||||
3. Run gnocchi-change-sack-size <number of sacks> to set new sack size. Note
|
||||
that sack value can only be changed if the backlog is empty.
|
||||
|
||||
4. Restart all gnocchi services (api, statsd, metricd) with new configuration
|
||||
|
||||
Alternatively, to minimise API downtime::
|
||||
|
||||
1. Run gnocchi-upgrade but use a new incoming storage target such as a new
|
||||
ceph pool, file path, etc... Additionally, set aggregate storage to a
|
||||
new target as well.
|
||||
|
||||
2. Run gnocchi-change-sack-size <number of sacks> against new target
|
||||
|
||||
3. Stop all input services (api, statsd)
|
||||
|
||||
4. Restart all input services but target newly created incoming storage
|
||||
|
||||
5. When done clearing backlog from original incoming storage, switch all
|
||||
metricd datemons to target new incoming storage but maintain original
|
||||
aggregate storage.
|
||||
|
||||
How to monitor Gnocchi
|
||||
======================
|
||||
|
||||
|
@ -76,6 +76,26 @@ def upgrade():
|
||||
index.create_archive_policy_rule("default", "*", "low")
|
||||
|
||||
|
||||
def change_sack_size():
|
||||
conf = cfg.ConfigOpts()
|
||||
conf.register_cli_opts([
|
||||
cfg.IntOpt("sack_size", required=True, min=1,
|
||||
help="Number of sacks."),
|
||||
])
|
||||
conf = service.prepare_service(conf=conf)
|
||||
s = storage.get_driver(conf)
|
||||
report = s.incoming.measures_report(details=False)
|
||||
remainder = report['summary']['measures']
|
||||
if remainder:
|
||||
LOG.error('Cannot change sack when non-empty backlog. Process '
|
||||
'remaining %s measures and try again', remainder)
|
||||
return
|
||||
LOG.info("Changing sack size to: %s", conf.sack_size)
|
||||
old_num_sacks = s.incoming.get_storage_sacks()
|
||||
s.incoming.set_storage_settings(conf.sack_size)
|
||||
s.incoming.remove_sack_group(old_num_sacks)
|
||||
|
||||
|
||||
def statsd():
|
||||
statsd_service.start()
|
||||
|
||||
|
@ -67,6 +67,10 @@ class CarbonaraBasedStorage(incoming.StorageDriver):
|
||||
def set_storage_settings(num_sacks):
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def remove_sack_group(num_sacks):
|
||||
raise NotImplementedError
|
||||
|
||||
def _unserialize_measures(self, measure_id, data):
|
||||
nb_measures = len(data) // self._MEASURE_SERIAL_LEN
|
||||
try:
|
||||
|
@ -69,6 +69,14 @@ class CephStorage(_carbonara.CarbonaraBasedStorage):
|
||||
self.ioctx.write_full(self.CFG_PREFIX,
|
||||
json.dumps({self.CFG_SACKS: num_sacks}).encode())
|
||||
|
||||
def remove_sack_group(self, num_sacks):
|
||||
prefix = self.get_sack_prefix(num_sacks)
|
||||
for i in six.moves.xrange(num_sacks):
|
||||
try:
|
||||
self.ioctx.remove_object(prefix % i)
|
||||
except rados.ObjectNotFound:
|
||||
pass
|
||||
|
||||
def add_measures_batch(self, metrics_and_measures):
|
||||
names_by_sack = defaultdict(list)
|
||||
for metric, measures in six.iteritems(metrics_and_measures):
|
||||
|
@ -16,6 +16,7 @@ import datetime
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
@ -52,6 +53,11 @@ class FileStorage(_carbonara.CarbonaraBasedStorage):
|
||||
utils.ensure_paths([self._sack_path(i)
|
||||
for i in six.moves.range(self.NUM_SACKS)])
|
||||
|
||||
def remove_sack_group(self, num_sacks):
|
||||
prefix = self.get_sack_prefix(num_sacks)
|
||||
for i in six.moves.xrange(num_sacks):
|
||||
shutil.rmtree(os.path.join(self.basepath, prefix % i))
|
||||
|
||||
def _sack_path(self, sack):
|
||||
return os.path.join(self.basepath, self.get_sack_name(sack))
|
||||
|
||||
|
@ -34,6 +34,11 @@ class RedisStorage(_carbonara.CarbonaraBasedStorage):
|
||||
def set_storage_settings(self, num_sacks):
|
||||
self._client.hset(self.CFG_PREFIX, self.CFG_SACKS, num_sacks)
|
||||
|
||||
@staticmethod
|
||||
def remove_sack_group(num_sacks):
|
||||
# NOTE(gordc): redis doesn't maintain keys with empty values
|
||||
pass
|
||||
|
||||
def _build_measure_path(self, metric_id):
|
||||
return redis.SEP.join([
|
||||
self.get_sack_name(self.sack_for_metric(metric_id)),
|
||||
|
@ -59,6 +59,11 @@ class S3Storage(_carbonara.CarbonaraBasedStorage):
|
||||
# NOTE(gordc): override to follow s3 partitioning logic
|
||||
return '%s-' + ('%s/' % (num_sacks if num_sacks else self.NUM_SACKS))
|
||||
|
||||
@staticmethod
|
||||
def remove_sack_group(num_sacks):
|
||||
# nothing to cleanup since sacks are part of path
|
||||
pass
|
||||
|
||||
def upgrade(self, indexer, num_sacks):
|
||||
try:
|
||||
s3.create_bucket(self.s3, self._bucket_name_measures,
|
||||
|
@ -46,6 +46,11 @@ class SwiftStorage(_carbonara.CarbonaraBasedStorage):
|
||||
for i in six.moves.range(self.NUM_SACKS):
|
||||
self.swift.put_container(self.get_sack_name(i))
|
||||
|
||||
def remove_sack_group(self, num_sacks):
|
||||
prefix = self.get_sack_prefix(num_sacks)
|
||||
for i in six.moves.xrange(num_sacks):
|
||||
self.swift.delete_container(prefix % i)
|
||||
|
||||
def _store_new_measures(self, metric, data):
|
||||
now = datetime.datetime.utcnow().strftime("_%Y%m%d_%H:%M:%S")
|
||||
self.swift.put_object(
|
||||
|
@ -133,6 +133,7 @@ gnocchi.rest.auth_helper =
|
||||
console_scripts =
|
||||
gnocchi-config-generator = gnocchi.cli:config_generator
|
||||
gnocchi-upgrade = gnocchi.cli:upgrade
|
||||
gnocchi-change-sack-size = gnocchi.cli:change_sack_size
|
||||
gnocchi-statsd = gnocchi.cli:statsd
|
||||
gnocchi-metricd = gnocchi.cli:metricd
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user