openstack/charm-trilio-data-mover
Code Issues Proposed changes

Faily major refactor

Browse Source
This commit is contained in:
James Page 2020-03-09 15:17:45 +00:00
parent c0d5e1795b
commit 352640c1d2
15 changed files with 262 additions and 772 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.gitignore vendored
View File

@ -1,11 +1,8 @@
bin
.coverage
.testrepository
.tox
*.sw[nop]
*.pyc
.unit-state.db
.stestr
__pycache__
build/
.local/
.testrepository/
.tox/
func-results.json
tests/id_rsa_zaza
test-charm/
**/__pycache__
.stestr

32
src/config.yaml
View File

@ -12,30 +12,6 @@ options:
type: string
default:
description: Downloadable URL of triliovault contego virtual environment
tvault-datamover-ext-usr:
type: string
default: nova
description: nova service user name
tvault-datamover-ext-group:
type: string
default: nova
description: nova service group name
tvault-datamover-virtenv:
type: string
default: /home/tvault
description: Trilio Vault home directory
tvault-datamover-virtenv-path:
type: string
default: /home/tvault/.virtenv
description: Trilio Vault Datamover virtual env
tv-datamover-conf:
type: string
default: /etc/tvault-contego/tvault-contego.conf
description: Trilio Vault Datamover config file location
nova-config:
type: string
default: /etc/nova/nova.conf
description: Nova default configuration file location
backup-target-type:
type: string
default: nfs
@ -80,14 +56,6 @@ options:
type: string
default:
description: S3 endpoint URL
tv-datamover-debug:
type: boolean
default: False
description: debug parameter value in /etc/tvault-contego/tvault-contego.conf
tv-datamover-verbose:
type: boolean
default: True
description: verbose parameter value in /etc/tvault-contego/tvault-contego.conf
tv-datamover-max-uploads-pending:
type: int
default: 3

22
src/files/trilio/get_nova_conf.py
View File

@ -1,22 +0,0 @@
from oslo_config import cfg
from nova import config as nova_conf
import sys
CONF = cfg.CONF
default_config_files = sys.argv[1].split(',') if len(
sys.argv) > 1 else ['/etc/nova/nova.conf']
nova_conf.parse_args(["/usr/bin/nova-compute"])
if not ('config_file' in CONF.keys() and CONF['config_file']):
try:
nova_conf.parse_args(
["/usr/bin/nova-compute"],
default_config_files=default_config_files)
except cfg.ConfigFilesNotFoundError:
raise
except BaseException:
pass
config_files = " --config-file=".join([""] + CONF['config_file']).strip()
print(config_files)

2
src/files/trilio/get_pkgs.py
View File

@ -1,2 +0,0 @@
from distutils.sysconfig import get_python_lib
print(get_python_lib())

3
src/files/trilio/trilio_sudoers
View File

@ -1 +1,2 @@
nova ALL = (root) NOPASSWD: /home/tvault/.virtenv/bin/privsep-helper *
# This is a security issue and needs changing
nova ALL=(ALL) NOPASSWD: ALL

14
src/files/trilio/tvault-contego
View File

@ -1,9 +1,9 @@
/var/log/nova/tvault-contego.log {
daily
missingok
notifempty
copytruncate
size=25M
rotate 3
compress
}
missingok
notifempty
copytruncate
size=25M
rotate 3
compress
}

16
src/files/trilio/tvault-contego.service Normal file
View File

@ -0,0 +1,16 @@
[Unit]
Description = TrilioVault DataMover
After = nova-compute.service
[Service]
User = nova
Group = nova
Type = simple
ExecStart = /usr/bin/tvault-contego --config-file=/etc/nova/nova.conf --config-file=/etc/tvault-contego/tvault-contego.conf
MemoryMax = 10G
TimeoutStopSec = 20
KillMode = process
Restart = always
[Install]
WantedBy = multi-user.target

0
src/files/trilio/tvault-object-store.service Normal file
View File

84
src/files/trilio/validate_s3.py
View File

@ -1,84 +0,0 @@
#!/usr/bin/python
import os
import boto3
import botocore
import argparse
from urllib.parse import urlparse
def validate_s3_credentials(s3_access_key_id, s3_secret_access_key,
s3_endpoint, s3_region, s3_bucket,
use_ssl, s3_signature_version):
""" Validate the S3 credentials.
Validate all of the S3 credentials by attempting to get
some bucket information.
Returns:
Success will be returned otherwise error 403, 404, or
500 will be retured with any relevent information.
"""
s3_config_object = None
if s3_signature_version != 'default' and s3_signature_version != '':
s3_config_object = botocore.client.Config(
signature_version=s3_signature_version)
s3_client = boto3.client('s3',
region_name=s3_region,
use_ssl=use_ssl,
aws_access_key_id=s3_access_key_id,
aws_secret_access_key=s3_secret_access_key,
endpoint_url=s3_endpoint,
config=s3_config_object)
s3_client.head_bucket(Bucket=s3_bucket)
# Add a check to see if the current object store will support
# our path length.
long_key = os.path.join(
'tvault_config/',
'workload_f5190be6-7f80-4856-8c24-149cb40500c5/',
'snapshot_f2e5c6a7-3c21-4b7f-969c-915bb408c64f/',
'vm_id_e81d1ac8-b49a-4ccf-9d92-5f1ef358f1be/',
'vm_res_id_72477d99-c475-4a5d-90ae-2560f5f3b319_vda/',
'deac2b8a-dca9-4415-adc1-f3c6598204ed-segments/',
'0000000000000000.00000000')
s3_client.put_object(
Bucket=s3_bucket, Key=long_key, Body='Test Data')
s3_client.delete_object(Bucket=s3_bucket, Key=long_key)
return {'status': 'Success'}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--access-key', required=True)
parser.add_argument('-s', '--secret-key', required=True)
parser.add_argument('-e', '--endpoint-url', default=None)
parser.add_argument('-b', '--bucket-name', required=True)
parser.add_argument('-r', '--region-name', default='us-east-2')
parser.add_argument('-v', '--signature-version', default='default')
args = parser.parse_args()
s3_access_key_id = args.access_key
s3_secret_access_key = args.secret_key
s3_endpoint = args.endpoint_url if args.endpoint_url else None
use_ssl = True if (s3_endpoint and
urlparse(s3_endpoint).scheme == 'https') else False
s3_region = args.region_name if args.region_name else None
s3_bucket = args.bucket_name
s3_signature_version = args.signature_version
try:
validate_s3_credentials(s3_access_key_id, s3_secret_access_key,
s3_endpoint, s3_region, s3_bucket,
use_ssl, s3_signature_version)
except Exception:
raise
main()

3
src/layer.yaml
View File

@ -1,2 +1 @@
---
includes: ['layer:basic'] # if you use any interfaces, add them here
includes: ['layer:openstack', 'interface:rabbitmq']

133
src/lib/charm/openstack/trilio_dm.py Normal file
View File

@ -0,0 +1,133 @@
# Copyright 2020 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import subprocess
import shutil
import charmhelpers.core.hookenv as hookenv
import charmhelpers.core.host as ch_host
import charms_openstack.charm
import charms_openstack.adapters as os_adapters
DM_USR = 'nova'
DM_GRP = 'nova'
VALID_BACKUP_TARGETS = [
'nfs',
's3'
]
class TrilioDataMoverCharm(charms_openstack.charm.OpenStackCharm):
service_name = name = "trilio-data-mover"
adapters_class = os_adapters.OpenStackAPIRelationAdapters
data_mover_conf = '/etc/tvault-contego/tvault-contego.conf'
logrotate_conf = "/etc/logrotate.d/tvault-contego"
# First release supported
release = "stein"
service_type = "data-mover"
default_service = "tvault-contego"
required_relations = ["amqp"]
package_codenames = {
"tvault-contego": collections.OrderedDict([("3", "stein")]),
"python3-tvault-contego": collections.OrderedDict([("3", "stein")]),
}
# configuration file permissions
user = "root"
group = DM_GRP
def get_amqp_credentials(self):
return ("datamover", "openstack")
def configure_source(self):
with open("/etc/apt/sources.list.d/trilio-wlm.list", "w") as tsources:
tsources.write(hookenv.config("triliovault-pkg-source"))
super().configure_source()
@property
def packages(self):
if hookenv.config('python-version') == 2:
return ['tvault-contego', 'nfs-common']
return ['python3-tvault-contego', 'nfs-common']
@property
def services(self):
if hookenv.config('backup-target-type') == 's3':
return ["tvault-contego", "tvault-object-store"]
return ["tvault-contego"]
@property
def restart_map(self):
return {
self.data_mover_conf: self.services,
}
# TODO: drop once packaging is updated
def install(self):
super().install()
self.ensure_dirs()
self.install_files()
self.configure_nova_user()
# TODO: drop once included in packages
def ensure_dirs(self):
"""
Ensures all the required directories are present
and have appropriate permissions.
"""
ch_host.mkdir(hookenv.config('tv-data-dir'),
owner=DM_USR, group=DM_GRP, perms=0o770, force=True)
# TODO: review this?
# os.system('rm -rf {}'.format(hookenv.config('tv-data-dir-old')))
ch_host.mkdir(hookenv.config('tv-data-dir-old'),
owner=DM_USR, group=DM_GRP, perms=0o770, force=True)
ch_host.mkdir('/etc/tvault-contego',
owner='root', group=DM_GRP, perms=0o750, force=True)
# TODO: drop once included in packages
def install_files(self):
"""
Installs a load of files that should be provided
by the package
"""
# "files/trilio/tvault-object-store.service":
# "/etc/systemd/system",
_file_map = {
"files/trilio/tvault-contego.service":
"/etc/systemd/system",
"files/trilio/trilio.filters":
"/etc/nova/rootwrap.d",
"files/trilio/trilio_sudoers":
"/etc/sudoers.d/",
"files/trilio/tvault-contego":
"/etc/logrotate.d/",
}
for file, target in _file_map.items():
shutil.copy(file, target)
# TODO: review why this is required
def configure_nova_user(self):
"""Add nova user to kvm and disk groups"""
for grp in ('kvm', 'disk'):
ch_host.add_user_to_group(DM_USR, grp)

49
src/reactive/data_mover_handlers.py Normal file
View File

@ -0,0 +1,49 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import charms_openstack.charm as charm
import charms.reactive as reactive
# This charm's library contains all of the handler code associated with
# trilio_dm
import charm.openstack.trilio_dm as trilio_dm # noqa
charm.use_defaults(
"charm.installed",
"config.changed",
"update-status",
)
@reactive.when("amqp.available")
def render_config(*args):
"""Render the configuration for charm when all the interfaces are
available.
"""
with charm.provide_charm_instance() as charm_class:
charm_class.render_with_interfaces(args)
charm_class.assess_status()
reactive.set_state("config.rendered")
# NOTE(jamespage): default handler is in api layer which is to much
@reactive.when('amqp.connected')
def default_amqp_connection(amqp):
"""Handle the default amqp connection.
This requires that the charm implements get_amqp_credentials() to
provide a tuple of the (user, vhost) for the amqp server
"""
with charm.provide_charm_instance() as instance:
user, vhost = instance.get_amqp_credentials()
amqp.request_access(username=user, vhost=vhost)
instance.assess_status()

610
src/reactive/trilio_data_mover.py
View File

@ -1,610 +0,0 @@
import os
import re
import configparser
import time
from subprocess import (
check_output,
call,
)
from charms.reactive import (
when,
when_not,
set_flag,
clear_flag,
hook,
remove_state,
set_state,
)
from charmhelpers.core.hookenv import (
status_set,
config,
log,
application_version_set,
)
from charmhelpers.fetch import (
apt_install,
apt_update,
apt_purge,
filter_missing_packages,
)
from charmhelpers.core.host import (
service_restart,
service_stop,
service_running,
write_file,
mount,
umount,
mounts,
add_user_to_group,
symlink,
mkdir,
chownr,
)
VALID_BACKUP_TARGETS = [
'nfs',
's3'
]
def get_new_version(pkg_name):
"""
Get the latest version available on the TrilioVault node.
"""
apt_cmd = "apt list {}".format(pkg_name)
pkg = check_output(apt_cmd.split()).decode('utf-8')
new_ver = re.search(r'\s([\d.]+)', pkg).group().strip()
return new_ver
def check_presence(tv_file):
"""
Just a wrpper of 'ls' command
"""
if os.system('ls {}'.format(tv_file)):
return False
return True
def validate_nfs():
"""
Validate the nfs mount device
"""
usr = config('tvault-datamover-ext-usr')
grp = config('tvault-datamover-ext-group')
data_dir = config('tv-data-dir')
device = config('nfs-shares')
nfs_options = config('nfs-options')
# install nfs-common package
if not filter_missing_packages(['nfs-common']):
log("'nfs-common' package not found, installing the package...")
apt_install(['nfs-common'], fatal=True)
if not device:
log("NFS mount device can not be empty."
"Check 'nfs-shares' value in config")
return False
# Ensure mount directory exists
mkdir(data_dir, owner=usr, group=grp, perms=501, force=True)
# check for mountable device
if not mount(device, data_dir, options=nfs_options, filesystem='nfs'):
log("Unable to mount, please enter valid mount device")
return False
log("Device mounted successfully")
umount(data_dir)
log("Device unmounted successfully")
return True
def validate_s3():
"""
Validate S3 backup target
"""
s3_access_key = config('tv-s3-access-key')
s3_secret_key = config('tv-s3-secret-key')
s3_endpoint = config('tv-s3-endpoint-url')
s3_bucket = config('tv-s3-bucket')
s3_region = config('tv-s3-region-name')
if not s3_access_key or not s3_secret_key:
log("Empty values provided!")
return False
if not s3_endpoint:
s3_endpoint = ''
if not s3_region:
s3_region = ''
cmd = ['python', 'files/trilio/validate_s3.py',
'-a', s3_access_key,
'-s', s3_secret_key,
'-e', s3_endpoint,
'-b', s3_bucket,
'-r', s3_region]
if not call(cmd):
log("Valid S3 credentials")
return True
log("Invalid S3 credentials")
return False
def validate_backup():
"""
Forwards to the respective modules accroding to the type of backup target.
"""
bkp_type = config('backup-target-type').lower()
if bkp_type not in VALID_BACKUP_TARGETS:
log("Not a valid backup target type")
return False
if bkp_type == 'nfs':
return validate_nfs()
elif bkp_type == 's3':
return validate_s3()
def add_users():
"""
Adding passwordless sudo access to nova user and adding to required groups
"""
usr = config('tvault-datamover-ext-usr')
path = '/etc/sudoers.d/tvault-nova'
source = '/usr/lib'
destination = '/usr/lib64'
content = '{} ALL=(ALL) NOPASSWD: ALL'.format(usr)
try:
write_file(path, content, owner='root', group='root', perms=501)
# Adding nova user to system groups
add_user_to_group(usr, 'kvm')
add_user_to_group(usr, 'disk')
# create symlink /usr/lib64/
symlink(source, destination)
except Exception as e:
log("Failed while adding user with msg: {}".format(e))
return False
return True
def create_virt_env(pkg_name):
"""
Checks if latest version is installed or else imports the new virtual env
And installs the Datamover package.
"""
usr = config('tvault-datamover-ext-usr')
grp = config('tvault-datamover-ext-group')
path = config('tvault-datamover-virtenv')
dm_ver = None
# create virtenv dir(/home/tvault) if it does not exist
mkdir(path, owner=usr, group=grp, perms=501, force=True)
latest_dm_ver = get_new_version(pkg_name)
if dm_ver == latest_dm_ver:
log("Latest TrilioVault DataMover package is already installed,"
" exiting")
return True
# Install TrilioVault Datamover package
if not install_plugin(pkg_name):
return False
# change virtenv dir(/home/tvault) users to nova
chownr(path, usr, grp)
# Copy Trilio sudoers and filters files
os.system(
'cp files/trilio/trilio_sudoers /etc/sudoers.d/')
os.system(
'cp files/trilio/trilio.filters /etc/nova/rootwrap.d/')
return True
def ensure_files():
"""
Ensures all the required files or directories
are present before it starts the datamover service.
"""
usr = config('tvault-datamover-ext-usr')
grp = config('tvault-datamover-ext-group')
dm_bin = '/usr/bin/tvault-contego'
log_path = '/var/log/nova'
log_file = '{}/tvault-contego.log'.format(log_path)
conf_path = '/etc/tvault-contego'
# Creates log directory if doesn't exists
mkdir(log_path, owner=usr, group=grp, perms=501, force=True)
write_file(log_file, '', owner=usr, group=grp, perms=501)
if not check_presence(dm_bin):
log("TrilioVault Datamover binary is not present")
return False
# Creates conf directory if doesn't exists
mkdir(conf_path, owner=usr, group=grp, perms=501, force=True)
return True
def create_conf():
"""
Creates datamover config file.
"""
nfs_share = config('nfs-shares')
nfs_options = config('nfs-options')
tv_data_dir_old = config('tv-data-dir-old')
tv_data_dir = config('tv-data-dir')
bkp_type = config('backup-target-type')
tv_config = configparser.RawConfigParser()
if bkp_type == 'nfs':
tv_config.set('DEFAULT', 'vault_storage_nfs_export', nfs_share)
tv_config.set('DEFAULT', 'vault_storage_nfs_options', nfs_options)
elif bkp_type == 's3':
tv_config.set('DEFAULT', 'vault_storage_nfs_export', 'TrilioVault')
tv_config.set('DEFAULT', 'vault_s3_auth_version', 'DEFAULT')
tv_config.set('DEFAULT', 'vault_s3_access_key_id',
config('tv-s3-access-key'))
tv_config.set('DEFAULT', 'vault_s3_secret_access_key',
config('tv-s3-secret-key'))
tv_config.set('DEFAULT', 'vault_s3_region_name',
config('tv-s3-region-name') or '')
tv_config.set('DEFAULT', 'vault_s3_bucket', config('tv-s3-bucket'))
tv_config.set('DEFAULT', 'vault_s3_endpoint_url',
config('tv-s3-endpoint-url') or '')
tv_config.set('DEFAULT', 'vault_storage_type', bkp_type)
tv_config.set('DEFAULT', 'vault_data_directory_old', tv_data_dir_old)
tv_config.set('DEFAULT', 'vault_data_directory', tv_data_dir)
tv_config.set('DEFAULT', 'log_file', '/var/log/nova/tvault-contego.log')
tv_config.set('DEFAULT', 'debug', config('tv-datamover-debug'))
tv_config.set('DEFAULT', 'verbose', config('tv-datamover-verbose'))
tv_config.set('DEFAULT', 'max_uploads_pending',
config('tv-datamover-max-uploads-pending'))
tv_config.set('DEFAULT', 'max_commit_pending',
config('tv-datamover-max-commit-pending'))
tv_config.set('DEFAULT', 'qemu_agent_ping_timeout',
config('tv-datamover-qemu-agent-ping-timeout'))
tv_config.add_section('contego_sys_admin')
tv_config.set('contego_sys_admin', 'helper_command',
'sudo /usr/bin/privsep-helper')
tv_config.add_section('conductor')
tv_config.set('conductor', 'use_local', True)
with open(config('tv-datamover-conf'), 'w') as cf:
tv_config.write(cf)
return True
def ensure_data_dir():
"""
Ensures all the required directories are present
and have appropriate permissions.
"""
usr = config('tvault-datamover-ext-usr')
grp = config('tvault-datamover-ext-group')
data_dir = config('tv-data-dir')
data_dir_old = config('tv-data-dir-old')
# ensure that data_dir is present
mkdir(data_dir, owner=usr, group=grp, perms=501, force=True)
# remove data_dir_old
os.system('rm -rf {}'.format(data_dir_old))
# recreate the data_dir_old
mkdir(data_dir_old, owner=usr, group=grp, perms=501, force=True)
# create logrotate file for tvault-contego.log
src = 'files/trilio/tvault-contego'
dest = '/etc/logrotate.d/tvault-contego'
os.system('cp {} {}'.format(src, dest))
return True
def create_service_file():
"""
Creates datamover service file.
"""
usr = config('tvault-datamover-ext-usr')
grp = config('tvault-datamover-ext-group')
usr_nova_conf = config('nova-config')
if not os.path.isfile(usr_nova_conf):
log("Try providing the correct path of nova.conf in config param")
status_set(
'blocked',
'Failed to find nova.conf file"')
return False
config_files = '--config-file={} --config-file={}'.format(
usr_nova_conf, config('tv-datamover-conf'))
if check_presence('/etc/nova/nova.conf.d'):
config_files = '{} --config-dir=/etc/nova/nova.conf.d'.format(
config_files)
# create service file
exec_start = '/usr/bin/python{} /usr/bin/tvault-contego {}\
'.format(config('python-version'), config_files)
tv_config = configparser.RawConfigParser()
tv_config.optionxform = str
tv_config.add_section('Unit')
tv_config.add_section('Service')
tv_config.add_section('Install')
tv_config.set('Unit', 'Description', 'TrilioVault DataMover')
tv_config.set('Unit', 'After', 'openstack-nova-compute.service')
tv_config.set('Service', 'User', usr)
tv_config.set('Service', 'Group', grp)
tv_config.set('Service', 'Type', 'simple')
tv_config.set('Service', 'ExecStart', exec_start)
tv_config.set('Service', 'MemoryMax', '10G')
tv_config.set('Service', 'TimeoutStopSec', 20)
tv_config.set('Service', 'KillMode', 'process')
tv_config.set('Service', 'Restart', 'always')
tv_config.set('Install', 'WantedBy', 'multi-user.target')
with open('/etc/systemd/system/tvault-contego.service', 'w') as cf:
tv_config.write(cf)
return True
def create_object_storage_service():
"""
Creates object storage service file.
"""
usr = config('tvault-datamover-ext-usr')
grp = config('tvault-datamover-ext-group')
venv_path = config('tvault-datamover-virtenv-path')
# Get dependent libraries paths
try:
cmd = ['/usr/bin/python{}'.format(config('python-version')),
'files/trilio/get_pkgs.py']
contego_path = check_output(cmd).decode('utf-8').strip()
except Exception as e:
log("Failed to get the dependent packages--{}".format(e))
return False
storage_path = '{}/contego/nova/extension/driver/s3vaultfuse.py'\
.format(contego_path)
config_file = config('tv-datamover-conf')
# create service file
exec_start = '{}/bin/python {} --config-file={}'\
.format(venv_path, storage_path, config_file)
tv_config = configparser.RawConfigParser()
tv_config.optionxform = str
tv_config.add_section('Unit')
tv_config.add_section('Service')
tv_config.add_section('Install')
tv_config.set('Unit', 'Description', 'TrilioVault Object Store')
tv_config.set('Unit', 'After', 'tvault-contego.service')
tv_config.set('Service', 'User', usr)
tv_config.set('Service', 'Group', grp)
tv_config.set('Service', 'Type', 'simple')
tv_config.set('Service', 'LimitNOFILE', 500000)
tv_config.set('Service', 'LimitNPROC', 500000)
tv_config.set('Service', 'ExecStart', exec_start)
tv_config.set('Service', 'TimeoutStopSec', 20)
tv_config.set('Service', 'KillMode', 'process')
tv_config.set('Service', 'Restart', 'on-failure')
tv_config.set('Install', 'WantedBy', 'multi-user.target')
with open('/etc/systemd/system/tvault-object-store.service', 'w') as cf:
tv_config.write(cf)
return True
def install_plugin(pkg_name):
"""
Install TrilioVault DataMover package
"""
try:
apt_install([pkg_name], ['--no-install-recommends'], fatal=True)
log("TrilioVault DataMover package installation passed")
status_set('maintenance', 'Starting...')
return True
except Exception as e:
# Datamover package installation failed
log("TrilioVault Datamover package installation failed")
log("With exception --{}".format(e))
return False
def uninstall_plugin(pkg_name):
"""
Uninstall TrilioVault DataMover packages
"""
retry_count = 0
bkp_type = config('backup-target-type')
try:
service_stop('tvault-contego')
os.system('sudo systemctl disable tvault-contego')
os.system('rm -rf /etc/systemd/system/tvault-contego.service')
if bkp_type == 's3':
service_stop('tvault-object-store')
os.system('systemctl disable tvault-object-store')
os.system('rm -rf /etc/systemd/system/tvault-object-store.service')
os.system('sudo systemctl daemon-reload')
os.system('rm -rf /etc/logrotate.d/tvault-contego')
os.system('rm -rf {}'.format(config('tv-datamover-conf')))
os.system('rm -rf /var/log/nova/tvault-contego.log')
# Get the mount points and un-mount tvault's mount points.
mount_points = mounts()
sorted_list = [mp[0] for mp in mount_points
if config('tv-data-dir') in mp[0]]
# stopping the tvault-object-store service may take time
while service_running('tvault-object-store') and retry_count < 3:
log('Waiting for tvault-object-store service to stop')
retry_count += 1
time.sleep(5)
for sl in sorted_list:
umount(sl)
# Uninstall tvault-contego package
apt_purge([pkg_name, 'contego'])
log("TrilioVault Datamover package uninstalled successfully")
return True
except Exception as e:
# package uninstallation failed
log("TrilioVault Datamover package un-installation failed:"
" {}".format(e))
return False
@when_not('tvault-contego.installed')
def install_tvault_contego_plugin():
status_set('maintenance', 'Installing...')
# Read config parameters
bkp_type = config('backup-target-type')
if config('python-version') == 2:
pkg_name = 'tvault-contego'
else:
pkg_name = 'python3-tvault-contego'
# add triliovault package repo
os.system('sudo echo "{}" > '
'/etc/apt/sources.list.d/trilio-gemfury-sources.list'.format(
config('triliovault-pkg-source')))
apt_update()
# Valildate backup target
if not validate_backup():
log("Failed while validating backup")
status_set(
'blocked',
'Invalid Backup target info, please provide valid info')
return
# Proceed as triliovault_ip Address is valid
if not add_users():
log("Failed while adding Users")
status_set('blocked', 'Failed while adding Users')
return
pkg_loc = create_virt_env(pkg_name)
if not pkg_loc:
log("Failed while Creating Virtual Env")
status_set('blocked', 'Failed while Creating Virtual Env')
return
if not ensure_files():
log("Failed while ensuring files")
status_set('blocked', 'Failed while ensuring files')
return
if not create_conf():
log("Failed while creating conf files")
status_set('blocked', 'Failed while creating conf files')
return
if not ensure_data_dir():
log("Failed while ensuring datat directories")
status_set('blocked', 'Failed while ensuring datat directories')
return
if not create_service_file():
log("Failed while creating DataMover service file")
status_set('blocked', 'Failed while creating DataMover service file')
return
if bkp_type == 's3' and not create_object_storage_service():
log("Failed while creating Object Store service file")
status_set('blocked', 'Failed while creating ObjectStore service file')
return
os.system('sudo systemctl daemon-reload')
# Enable and start the object-store service
if bkp_type == 's3':
os.system('sudo systemctl enable tvault-object-store')
service_restart('tvault-object-store')
# Enable and start the datamover service
os.system('sudo systemctl enable tvault-contego')
service_restart('tvault-contego')
# Install was successful
status_set('active', 'Ready...')
# Add the flag "installed" since it's done
application_version_set(get_new_version(pkg_name))
set_flag('tvault-contego.installed')
@hook('stop')
def stop_handler():
# Set the user defined "stopping" state when this hook event occurs.
set_state('tvault-contego.stopping')
@when('tvault-contego.stopping')
def stop_tvault_contego_plugin():
status_set('maintenance', 'Stopping...')
if config('python-version') == 2:
pkg_name = 'tvault-contego'
else:
pkg_name = 'python3-tvault-contego'
# add triliovault package repo
# Call the script to stop and uninstll TrilioVault Datamover
uninst_ret = uninstall_plugin(pkg_name)
if uninst_ret:
# Uninstall was successful
# Remove the state "stopping" since it's done
remove_state('tvault-contego.stopping')
@hook('upgrade-charm')
def upgrade_charm():
# check if installed contego pkg is python 2 or 3
if os.system('dpkg -s python3-tvault-contego | grep Status') == 0:
pkg_name = 'python3-tvault-contego'
else:
pkg_name = 'tvault-contego'
# Call the script to stop and uninstll TrilioVault Datamover
uninst_ret = uninstall_plugin(pkg_name)
if uninst_ret:
# Uninstall was successful, clear flag to re-install
clear_flag('tvault-contego.installed')
@hook('config-changed')
def reconfig_charm():
bkp_type = config('backup-target-type')
# Valildate backup target
if not validate_backup():
log("Failed while validating backup")
status_set(
'blocked',
'Invalid Backup target info, please provide valid info')
return
if not create_conf():
log("Failed while creating conf files")
status_set('blocked', 'Failed while creating conf files')
return
# Re-start the object-store service
if bkp_type == 's3':
service_restart('tvault-object-store')
# Re-start the datamover service
service_restart('tvault-contego')
# Reconfig successful
status_set('active', 'Ready...')

47
src/templates/tvault-contego.conf Normal file
View File

@ -0,0 +1,47 @@
# This file is managed by Juju - changes will be overwritten
[DEFAULT]
{% if amqp.transport_url -%}
# AMQP
transport_url = {{ amqp.transport_url }}
rabbit_virtual_host = {{ amqp.vhost }}
{% endif -%}
{% if options.backup_target_type == 'nfs' -%}
# NFS
vault_storage_nfs_export = {{ options.nfs_shares }}
vault_storage_nfs_options = {{ options.nfs_options }}
{% endif -%}
{% if options.backup_target_type == 's3' -%}
# S3
vault_storage_nfs_export = TrilioVault
vault_s3_auth_version = DEFAULT
vault_s3_access_key_id = {{ options.tv_s3_access_key }}
vault_s3_secret_access_key = {{ options.tv_s3_secret_key }}
vault_s3_region_name = {{ options.tv_s3_region_name }}
vault_s3_bucket = {{ options.tv_s3_bucket }}
vault_s3_endpoint_url = {{ options.tv_s3_endpoint_url }}
{% endif -%}
# MISC
vault_storage_type = {{ options.backup_target_type }}
vault_data_directory_old = {{ options.tv_data_dir_old }}
vault_data_directory = {{ options.tv_data_dir }}
# Logging
log_file = /var/log/nova/tvault-contego.log
debug = {{ options.debug }}
verbose = {{ options.verbose }}
# Upload Configuration
max_uploads_pending = {{ options.tv_datamover_max_uploads_pending }}
max_commit_pending = {{ options.tv_datamover_max_commit_pending }}
qemu_agent_ping_timeout = {{ options.tv_datamover_qemu_agent_ping_timeout }}
# privsep
[contego_sys_admin]
helper_command = sudo /usr/bin/privsep-helper
[conductor]
use_local = True

2
src/wheelhouse.txt
View File

@ -1,2 +0,0 @@
boto3
botocore