From faaee49a6f0fbd152a63b1b97a6cebb9926929eb Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 20 Jan 2015 15:11:51 +0000 Subject: [PATCH] Initial support for deploying from git. --- config.yaml | 16 ++++ config/git-juno-minimal.yaml | 6 ++ config/git-tip-all.yaml | 57 +++++++++++ config/git-tip-minimal.yaml | 6 ++ hooks/glance_relations.py | 32 ++++--- hooks/glance_utils.py | 160 +++++++++++++++++++++++++++++-- templates/upstart/glance.upstart | 11 +++ unit_tests/test_glance_utils.py | 6 +- 8 files changed, 275 insertions(+), 19 deletions(-) create mode 100644 config/git-juno-minimal.yaml create mode 100644 config/git-tip-all.yaml create mode 100644 config/git-tip-minimal.yaml create mode 100644 templates/upstart/glance.upstart diff --git a/config.yaml b/config.yaml index 87ec9bf9..1dccec21 100644 --- a/config.yaml +++ b/config.yaml @@ -14,6 +14,22 @@ options: Note that updating this setting to a source that is known to provide a later version of OpenStack will trigger a software upgrade. + + Note that when openstack-origin-git is specified, the packages + configured for that option will be installed from git rather + than from the openstack-origin repository. + openstack-origin-git: + default: None + type: "string" + description: | + Specifies a YAML file which lists the git repositories and branches + from which to install OpenStack. See config/git-*.yaml for a + starting point. + + Note that the config files that are installed will be determined + based on the OpenStack release in the openstack-origin option. + + Note that updating this setting after deployment will do nothing. database-user: default: glance type: string diff --git a/config/git-juno-minimal.yaml b/config/git-juno-minimal.yaml new file mode 100644 index 00000000..d0998bd0 --- /dev/null +++ b/config/git-juno-minimal.yaml @@ -0,0 +1,6 @@ +glance: + repository: git://git.openstack.org/openstack/glance.git + branch: stable/juno +requirements: + repository: git://git.openstack.org/openstack/requirements.git + branch: stable/juno diff --git a/config/git-tip-all.yaml b/config/git-tip-all.yaml new file mode 100644 index 00000000..1350ecd0 --- /dev/null +++ b/config/git-tip-all.yaml @@ -0,0 +1,57 @@ +glance: + repository: git://git.openstack.org/openstack/glance.git + branch: master +glance-store: + repository: git://git.openstack.org/openstack/glance_store.git + branch: master +keystonemiddleware: + repository: git://git.openstack.org/openstack/keystonemiddleware.git + branch: master +oslo-config: + repository: git://git.openstack.org/openstack/oslo.config.git + branch: master +oslo-concurrency: + repository: git://git.openstack.org/openstack/oslo.concurrency.git + branch: master +oslo-db: + repository: git://git.openstack.org/openstack/oslo.db.git + branch: master +oslo-i18n: + repository: git://git.openstack.org/openstack/oslo.i18n.git + branch: master +oslo-messaging: + repository: git://git.openstack.org/openstack/oslo.messaging.git + branch: master +oslo-serialization: + repository: git://git.openstack.org/openstack/oslo.serialization.git + branch: master +oslo-utils: + repository: git://git.openstack.org/openstack/oslo.utils.git + branch: master +oslo-vmware: + repository: git://git.openstack.org/openstack/oslo.vmware.git + branch: master +osprofiler: + repository: git://git.openstack.org/stackforge/osprofiler.git + branch: master +pbr: + repository: git://git.openstack.org/openstack-dev/pbr.git + branch: master +python-keystoneclient: + repository: git://git.openstack.org/openstack/python-keystoneclient.git + branch: master +python-swiftclient: + repository: git://git.openstack.org/openstack/python-swiftclient.git + branch: master +requirements: + repository: git://git.openstack.org/openstack/requirements.git + branch: master +stevedore: + repository: git://git.openstack.org/openstack/stevedore.git + branch: master +sqlalchemy-migrate: + repository: git://git.openstack.org/stackforge/sqlalchemy-migrate.git + branch: master +wsme: + repository: git://git.openstack.org/stackforge/wsme.git + branch: master diff --git a/config/git-tip-minimal.yaml b/config/git-tip-minimal.yaml new file mode 100644 index 00000000..486c5306 --- /dev/null +++ b/config/git-tip-minimal.yaml @@ -0,0 +1,6 @@ +glance: + repository: git://git.openstack.org/openstack/glance.git + branch: master +requirements: + repository: git://git.openstack.org/openstack/requirements.git + branch: master diff --git a/hooks/glance_relations.py b/hooks/glance_relations.py index 04b2dc83..c21dedc1 100755 --- a/hooks/glance_relations.py +++ b/hooks/glance_relations.py @@ -7,12 +7,13 @@ import sys from glance_utils import ( do_openstack_upgrade, + git_install, migrate_database, register_configs, restart_map, services, CLUSTER_RES, - PACKAGES, + determine_packages, SERVICES, CHARM, GLANCE_REGISTRY_CONF, @@ -54,10 +55,11 @@ from charmhelpers.contrib.hahelpers.cluster import ( ) from charmhelpers.contrib.openstack.utils import ( configure_installation_source, - get_os_codename_package, - openstack_upgrade_available, + git_install_requested, lsb_release, - sync_db_with_multi_ipv6_addresses + openstack_upgrade_available, + os_release, + sync_db_with_multi_ipv6_addresses, ) from charmhelpers.contrib.storage.linux.ceph import ( ensure_ceph_keyring, @@ -100,7 +102,8 @@ def install_hook(): configure_installation_source(src) apt_update(fatal=True) - apt_install(PACKAGES, fatal=True) + apt_install(determine_packages(), fatal=True) + git_install(config('openstack-origin-git')) for service in SERVICES: service_stop(service) @@ -140,7 +143,7 @@ def pgsql_db_joined(): @hooks.hook('shared-db-relation-changed') @restart_on_change(restart_map()) def db_changed(): - rel = get_os_codename_package("glance-common") + rel = os_release('glance-common') if 'shared-db' not in CONFIGS.complete_contexts(): juju_log('shared-db relation incomplete. Peer not ready?') @@ -172,7 +175,7 @@ def db_changed(): @hooks.hook('pgsql-db-relation-changed') @restart_on_change(restart_map()) def pgsql_db_changed(): - rel = get_os_codename_package("glance-common") + rel = os_release('glance-common') if 'pgsql-db' not in CONFIGS.complete_contexts(): juju_log('pgsql-db relation incomplete. Peer not ready?') @@ -308,9 +311,10 @@ def config_changed(): sync_db_with_multi_ipv6_addresses(config('database'), config('database-user')) - if openstack_upgrade_available('glance-common'): - juju_log('Upgrading OpenStack release') - do_openstack_upgrade(CONFIGS) + if not git_install_requested(): + if openstack_upgrade_available('glance-common'): + juju_log('Upgrading OpenStack release') + do_openstack_upgrade(CONFIGS) open_port(9292) configure_https() @@ -324,6 +328,12 @@ def config_changed(): [cluster_joined(rid) for rid in relation_ids('cluster')] +#TODO(coreycb): For deploy from git support, need to implement action-set +# and action-get to trigger re-install of git-installed +# services. IIUC they'd be triggered via: +# juju do + + @hooks.hook('cluster-relation-joined') def cluster_joined(relation_id=None): for addr_type in ADDRESS_TYPES: @@ -352,7 +362,7 @@ def cluster_changed(): @hooks.hook('upgrade-charm') @restart_on_change(restart_map(), stopstart=True) def upgrade_charm(): - apt_install(filter_installed_packages(PACKAGES), fatal=True) + apt_install(filter_installed_packages(determine_packages()), fatal=True) configure_https() update_nrpe_config() CONFIGS.write_all() diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index f1d6139c..f101d9d4 100755 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -1,6 +1,7 @@ #!/usr/bin/python import os +import shutil import subprocess import glance_contexts @@ -14,21 +15,26 @@ from charmhelpers.fetch import ( add_source) from charmhelpers.core.hookenv import ( + charm_dir, config, log, relation_ids, service_name) from charmhelpers.core.host import ( + adduser, + add_group, + add_user_to_group, mkdir, service_stop, service_start, - lsb_release + lsb_release, + write_file, ) from charmhelpers.contrib.openstack import ( templating, - context, ) + context,) from charmhelpers.contrib.hahelpers.cluster import ( eligible_leader, @@ -38,7 +44,13 @@ from charmhelpers.contrib.openstack.alternatives import install_alternative from charmhelpers.contrib.openstack.utils import ( get_os_codename_install_source, get_os_codename_package, - configure_installation_source) + git_install_requested, + git_clone_and_install, + configure_installation_source, + os_release, +) + +from charmhelpers.core.templating import render CLUSTER_RES = "grp_glance_vips" @@ -46,8 +58,27 @@ PACKAGES = [ "apache2", "glance", "python-mysqldb", "python-swiftclient", "python-psycopg2", "python-keystone", "python-six", "uuid", "haproxy", ] +BASE_GIT_PACKAGES = [ + 'libxml2-dev', + 'libxslt1-dev', + 'python-dev', + 'python-pip', + 'python-setuptools', + 'zlib1g-dev', +] + SERVICES = [ - "glance-api", "glance-registry", ] + "glance-api", + "glance-registry", +] + +# ubuntu packages that should not be installed when deploying from git +GIT_PACKAGE_BLACKLIST = [ + 'glance', + 'python-swiftclient', + 'python-keystone', +] + CHARM = "glance" @@ -136,7 +167,7 @@ def register_configs(): # Register config files with their respective contexts. # Regstration of some configs may not be required depending on # existing of certain relations. - release = get_os_codename_package('glance-common', fatal=False) or 'essex' + release = os_release('glance-common') configs = templating.OSConfigRenderer(templates_dir=TEMPLATES, openstack_release=release) @@ -173,6 +204,18 @@ def register_configs(): return configs +def determine_packages(): + packages = [] + PACKAGES + + if git_install_requested(): + packages.extend(BASE_GIT_PACKAGES) + # don't include packages that will be installed from git + for p in GIT_PACKAGE_BLACKLIST: + packages.remove(p) + + return list(set(packages)) + + def migrate_database(): '''Runs glance-manage to initialize a new database or migrate existing @@ -201,7 +244,7 @@ def do_openstack_upgrade(configs): ] apt_update() apt_upgrade(options=dpkg_opts, fatal=True, dist=True) - apt_install(PACKAGES, fatal=True) + apt_install(packages=determine_packages(), fatal=True) # set CONFIGS to load templates from new release and regenerate config configs.set_release(openstack_release=new_os_rel) @@ -252,3 +295,108 @@ def setup_ipv6(): ' main') apt_update() apt_install('haproxy/trusty-backports', fatal=True) + + +def git_install(file_name): + """Perform setup, and install git repos specified in yaml config file.""" + if git_install_requested(): + git_pre_install() + git_clone_and_install(file_name, core_project='glance') + git_post_install() + + +def git_pre_install(): + """Perform pre glance installation setup.""" + dirs = [ + '/var/lib/glance', + '/var/lib/glance/images', + '/var/lib/glance/image-cache', + '/var/lib/glance/image-cache/incomplete', + '/var/lib/glance/image-cache/invalid', + '/var/lib/glance/image-cache/queue', + '/var/log/glance', + '/etc/glance', + ] + + logs = [ + '/var/log/glance/glance-api.log', + '/var/log/glance/glance-registry.log', + ] + + adduser('glance', shell='/bin/bash', system_user=True) + add_group('glance', system_group=True) + add_user_to_group('glance', 'glance') + + for d in dirs: + mkdir(d, owner='glance', group='glance', perms=0700, force=False) + + for l in logs: + write_file(l, '', owner='glance', group='glance', perms=0600) + + +def git_post_install(): + """Perform post glance installation setup.""" + src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/glance.git/etc/') + configs = { + 'glance-api-paste': { + 'src': os.path.join(src_etc, 'glance-api-paste.ini'), + 'dest': '/etc/glance/glance-api-paste.ini', + }, + 'glance-api': { + 'src': os.path.join(src_etc, 'glance-api.conf'), + 'dest': '/etc/glance/glance-api.conf', + }, + 'glance-registry-paste': { + 'src': os.path.join(src_etc, 'glance-registry-paste.ini'), + 'dest': '/etc/glance/glance-registry-paste.ini', + }, + 'glance-registry': { + 'src': os.path.join(src_etc, 'glance-registry.conf'), + 'dest': '/etc/glance/glance-registry.conf', + }, + 'glance-cache': { + 'src': os.path.join(src_etc, 'glance-cache.conf'), + 'dest': '/etc/glance/glance-cache.conf', + }, + 'glance-scrubber': { + 'src': os.path.join(src_etc, 'glance-scrubber.conf'), + 'dest': '/etc/glance/glance-scrubber.conf', + }, + 'policy': { + 'src': os.path.join(src_etc, 'policy.json'), + 'dest': '/etc/glance/policy.json', + }, + 'schema-image': { + 'src': os.path.join(src_etc, 'schema-image.json'), + 'dest': '/etc/glance/schema-image.json', + }, + } + + for conf, files in configs.iteritems(): + shutil.copyfile(files['src'], files['dest']) + + glance_api_context = { + 'service_description': 'Glance API server', + 'service_name': 'Glance', + 'user_name': 'glance', + 'start_dir': '/var/lib/glance', + 'process_name': 'glance-api', + 'executable_name': '/usr/local/bin/glance-api', + } + + glance_registry_context = { + 'service_description': 'Glance registry server', + 'service_name': 'Glance', + 'user_name': 'glance', + 'start_dir': '/var/lib/glance', + 'process_name': 'glance-registry', + 'executable_name': '/usr/local/bin/glance-registry', + } + + render('upstart/glance.upstart', '/etc/init/glance-api.conf', + glance_api_context, perms=0o644) + render('upstart/glance.upstart', '/etc/init/glance-registry.conf', + glance_registry_context, perms=0o644) + + service_start('glance-api') + service_start('glance-registry') diff --git a/templates/upstart/glance.upstart b/templates/upstart/glance.upstart new file mode 100644 index 00000000..e5f39741 --- /dev/null +++ b/templates/upstart/glance.upstart @@ -0,0 +1,11 @@ +description "{{ service_description }}" +author "Juju {{ service_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +exec start-stop-daemon --start --chuid {{ user_name }} \ + --chdir {{ start_dir }} --name {{ process_name }} \ + --exec {{ executable_name }} diff --git a/unit_tests/test_glance_utils.py b/unit_tests/test_glance_utils.py index 67d1105d..dcb3be03 100644 --- a/unit_tests/test_glance_utils.py +++ b/unit_tests/test_glance_utils.py @@ -130,7 +130,8 @@ class TestGlanceUtils(CharmTestCase): configs = MagicMock() utils.do_openstack_upgrade(configs) self.assertTrue(configs.write_all.called) - self.apt_install.assert_called_with(utils.PACKAGES, fatal=True) + self.apt_install.assert_called_with(utils.determine_packages(), + fatal=True) self.apt_upgrade.assert_called_with(options=DPKG_OPTS, fatal=True, dist=True) configs.set_release.assert_called_with(openstack_release='havana') @@ -145,7 +146,8 @@ class TestGlanceUtils(CharmTestCase): configs = MagicMock() utils.do_openstack_upgrade(configs) self.assertTrue(configs.write_all.called) - self.apt_install.assert_called_with(utils.PACKAGES, fatal=True) + self.apt_install.assert_called_with(utils.determine_packages(), + fatal=True) self.apt_upgrade.assert_called_with(options=DPKG_OPTS, fatal=True, dist=True) configs.set_release.assert_called_with(openstack_release='havana')