From 2c8b6c5e0b33cc9178d93bf82acaca64b6e61ea8 Mon Sep 17 00:00:00 2001
From: Simon Pasquier <spasquier@mirantis.com>
Date: Wed, 28 Sep 2016 15:46:55 +0200
Subject: [PATCH] Add the option to store the InfluxDB WAL in memory

This change adds a new option to the Fuel UI to store the InfluxDB WAL
files in memory. This greatly improves write performances but it may
lead to data loss in case of server crash.

The WAL partition will use 10% of the total RAM (but no more than 4GB).

DocImpact

Change-Id: I6153cd24322ef3eab53505bf4cb1f72d9594ec04
---
 deployment_scripts/puppet/manifests/hiera.pp  | 10 ++++++++
 .../puppet/manifests/influxdb.pp              | 23 +++++++++++++++++++
 .../manifests/influxdb.pp                     |  9 +++++++-
 environment_config.yaml                       |  7 ++++++
 4 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/deployment_scripts/puppet/manifests/hiera.pp b/deployment_scripts/puppet/manifests/hiera.pp
index 0e56f9c9..21c5e7bc 100644
--- a/deployment_scripts/puppet/manifests/hiera.pp
+++ b/deployment_scripts/puppet/manifests/hiera.pp
@@ -53,6 +53,14 @@ $influxdb_password = $influxdb_grafana['influxdb_userpass']
 $influxdb_dbname   = $influxdb_grafana['influxdb_dbname']
 
 $retention_period = $influxdb_grafana['retention_period']
+if $influxdb_grafana['influxdb_in_memory_wal'] {
+  $influxdb_wal_storage = 'memory'
+  # Allocate 10% of the total RAM for the WAL partition (but no more than 4GB)
+  $influxdb_wal_size = min(4 * 1024 * 1024 * 1024, $::memorysize_mb * 1024 * 0.1)
+} else {
+  $influxdb_wal_storage = 'disk'
+  $influxdb_wal_size = 0
+}
 
 # Parameters related to MySQL
 $host = $influxdb_grafana['mysql_host']
@@ -138,6 +146,8 @@ lma::corosync_roles:
 # from 1 or 2 nodes to 3 nodes.
 lma::influxdb::replication_factor: 3
 lma::influxdb::retention_period: <%= @retention_period %>
+lma::influxdb::wal::storage: <%= @influxdb_wal_storage %>
+lma::influxdb::wal::size: <%= @influxdb_wal_size %>
 
 lma::influxdb::admin_username: "root"
 lma::influxdb::admin_password: <%= @influxdb_admin_password %>
diff --git a/deployment_scripts/puppet/manifests/influxdb.pp b/deployment_scripts/puppet/manifests/influxdb.pp
index 6436f68f..3c5d0db7 100644
--- a/deployment_scripts/puppet/manifests/influxdb.pp
+++ b/deployment_scripts/puppet/manifests/influxdb.pp
@@ -15,6 +15,7 @@
 notice('fuel-plugin-influxdb-grafana: influxdb.pp')
 
 $data_directory = hiera('lma::influxdb::data_dir')
+$wal_dir = "${data_directory}/wal"
 
 # We set raft_nodes only for the non-primary node. The primary node will be
 # started as the first node and it will be the leader of the Raft cluster.
@@ -37,11 +38,33 @@ file { $data_directory:
   require => User['influxdb'],
 }
 
+if hiera('lma::influxdb::wal::storage') == 'memory' {
+  $wal_size = hiera('lma::influxdb::wal::size')
+  file { $wal_dir:
+    ensure  => directory,
+    owner   => 'influxdb',
+    group   => 'influxdb',
+    require => File[$data_directory],
+  }
+
+  mount { $wal_dir:
+    ensure   => mounted,
+    device   => 'tmpfs',
+    atboot   => true,
+    options  => "size=${wal_size},rw",
+    fstype   => 'tmpfs',
+    remounts => false,
+    before   => Class['lma_monitoring_analytics::influxdb'],
+    require  => File[$wal_dir],
+  }
+}
+
 # We cannot mix IP addresses and hostnames otherwise the Raft cluster won't
 # start. We have to stick with IP addresses because hostnames map to the
 # managament network space.
 class { 'lma_monitoring_analytics::influxdb':
   base_directory => $data_directory,
+  wal_dir        => $wal_dir,
   hostname       => hiera('lma::influxdb::listen_address'),
   raft_nodes     => $raft_nodes,
   version        => '0.11.1-1',
diff --git a/deployment_scripts/puppet/modules/lma_monitoring_analytics/manifests/influxdb.pp b/deployment_scripts/puppet/modules/lma_monitoring_analytics/manifests/influxdb.pp
index e3c35974..0944c859 100644
--- a/deployment_scripts/puppet/modules/lma_monitoring_analytics/manifests/influxdb.pp
+++ b/deployment_scripts/puppet/modules/lma_monitoring_analytics/manifests/influxdb.pp
@@ -16,6 +16,7 @@
 
 class lma_monitoring_analytics::influxdb (
   $base_directory = $lma_monitoring_analytics::params::influxdb_dir,
+  $wal_dir        = undef,
   $hostname       = undef,
   $raft_nodes     = undef,
   $version        = undef,
@@ -25,11 +26,17 @@ class lma_monitoring_analytics::influxdb (
     validate_array($raft_nodes)
   }
 
+  if $wal_dir {
+    $_wal_dir = $wal_dir
+  } else {
+    $_wal_dir = "${base_directory}/wal"
+  }
+
   class { '::influxdb':
     data_dir   => "${base_directory}/data",
     meta_dir   => "${base_directory}/meta",
     hh_dir     => "${base_directory}/hh",
-    wal_dir    => "${base_directory}/wal",
+    wal_dir    => $_wal_dir,
     snapshot   => true,
     hostname   => $hostname,
     raft_nodes => $raft_nodes,
diff --git a/environment_config.yaml b/environment_config.yaml
index 57c81a6a..ae8a4dd0 100644
--- a/environment_config.yaml
+++ b/environment_config.yaml
@@ -52,6 +52,13 @@ attributes:
       source: '^[\S]{4,}$'
       error: "You must provide a password with at least 4 characters"
 
+  influxdb_in_memory_wal:
+    value: false
+    label: 'Store WAL files in memory'
+    description: 'Store the Write-Ahead-Log (WAL) files in memory instead of disk. This will improve the write performances but data may be lost in case of server crash.'
+    weight: 61
+    type: "checkbox"
+
   grafana_username:
     value: 'lma'
     label: 'User name'