config: add environment variable substitution

This change enables setting configuration values through
environment variables. This is useful to manage user defined
configuration, such as user password, in Kubernetes deployment.

Change-Id: Iafbb63ebbb388ef3038f45fd3a929c3e7e2dc343
This commit is contained in:
Tristan Cacqueray 2020-05-20 11:44:49 +00:00
parent fa2a850cb9
commit eb9af85025
5 changed files with 31 additions and 13 deletions

View File

@ -11,11 +11,12 @@
# under the License.
import logging
import os
import voluptuous as v
import yaml
from nodepool.driver import ProviderConfig
from nodepool.config import get_provider_config
from nodepool.config import get_provider_config, substitute_env_vars
log = logging.getLogger(__name__)
@ -80,7 +81,7 @@ class ConfigValidator:
}
return v.Schema(top_level)
def validate(self):
def validate(self, env=os.environ):
'''
Validate a configuration file
@ -92,11 +93,13 @@ class ConfigValidator:
try:
with open(self.config_file) as f:
config = yaml.safe_load(f)
config = yaml.safe_load(substitute_env_vars(f.read(), env))
except Exception:
log.exception('YAML parsing failed')
return 1
self.config = config
try:
# validate the overall schema
ConfigValidator.getSchema()(config)

View File

@ -14,7 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import math
import os
import time
import yaml
@ -293,7 +295,16 @@ def get_provider_config(provider):
return driver.getProviderConfig(provider)
def openConfig(path):
def substitute_env_vars(config_str, env):
return functools.reduce(
lambda config, env_item: config.replace(
"%(" + env_item[0] + ")", env_item[1]),
[(k, v) for k, v in env.items()
if k.startswith('NODEPOOL_')],
config_str)
def openConfig(path, env):
retry = 3
# Since some nodepool code attempts to dynamically re-read its config
@ -303,8 +314,7 @@ def openConfig(path):
while True:
try:
with open(path) as f:
config = yaml.safe_load(f)
break
return yaml.safe_load(substitute_env_vars(f.read(), env))
except IOError as e:
if e.errno == 2:
retry = retry - 1
@ -313,11 +323,10 @@ def openConfig(path):
raise e
if retry == 0:
raise e
return config
def loadConfig(config_path):
config = openConfig(config_path)
def loadConfig(config_path, env=os.environ):
config = openConfig(config_path, env)
# Call driver config reset now to clean global hooks like openstacksdk
for driver in Drivers.drivers.values():
@ -347,8 +356,8 @@ def loadConfig(config_path):
return newconfig
def loadSecureConfig(config, secure_config_path):
secure = openConfig(secure_config_path)
def loadSecureConfig(config, secure_config_path, env=os.environ):
secure = openConfig(secure_config_path, env)
if not secure: # empty file
return

View File

@ -2,7 +2,7 @@ elements-dir: /etc/nodepool/elements
images-dir: /opt/nodepool_dib
webapp:
port: 8005
port: %(NODEPOOL_PORT)
listen_address: '0.0.0.0'
zookeeper-servers:

View File

@ -28,8 +28,9 @@ class TestConfigValidation(tests.BaseTestCase):
'fixtures', 'config_validate', 'good.yaml')
validator = ConfigValidator(config)
ret = validator.validate()
ret = validator.validate(dict(NODEPOOL_PORT="8005"))
self.assertEqual(ret, 0)
self.assertEqual(validator.config['webapp']['port'], 8005)
def test_yaml_error(self):
config = os.path.join(os.path.dirname(tests.__file__),

View File

@ -0,0 +1,5 @@
---
features:
- |
Configuration value can be set from the envirnonment variables using the
`%(NODEPOOL_env_name)` syntax.