diff --git a/README.md b/README.md index e73aafd9..973e2251 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,101 @@ This charm also supports scale out and high availability using the hacluster cha juju set neutron-api vip= juju add-relation neutron-hacluster neutron-api +# Deploying from source + +The minimal openstack-origin-git config required to deploy from source is: + + openstack-origin-git: + "{'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'stable/icehouse'}}" + +If you specify a 'requirements' repository, it will be used to update the +requirements.txt files of all other git repos that it applies to, before +they are installed: + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'master'}}" + +Note that there are only two key values the charm knows about for the outermost +dictionary: 'neutron' and 'requirements'. These repositories must correspond to +these keys. If the requirements repository is specified, it will be installed +first. The neutron repository is always installed last. All other repostories +will be installed in between. + +NOTE(coreycb): The following is temporary to keep track of the full list of +current tip repos (may not be up-to-date). + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'neutron-fwaas': + {'repository': 'git://git.openstack.org/openstack/neutron-fwaas.git', + 'branch': 'master'}, + 'neutron-lbaas': + {'repository: 'git://git.openstack.org/openstack/neutron-lbaas.git', + 'branch': 'master'}, + 'neutron-vpnaas': + {'repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git', + 'branch': 'master'}, + 'keystonemiddleware: + {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git', + 'branch: 'master'}, + 'oslo-concurrency': + {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git', + 'branch: 'master'}, + 'oslo-config': + {'repository': 'git://git.openstack.org/openstack/oslo.config.git', + 'branch: 'master'}, + 'oslo-context': + {'repository': 'git://git.openstack.org/openstack/oslo.context.git', + 'branch: 'master'}, + 'oslo-db': + {'repository': 'git://git.openstack.org/openstack/oslo.db.git', + 'branch: 'master'}, + 'oslo-i18n': + {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git', + 'branch: 'master'}, + 'oslo-messaging': + {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git', + 'branch: 'master'}, + 'oslo-middleware: + {'repository': 'git://git.openstack.org/openstack/oslo.middleware.git', + 'branch': 'master'}, + 'oslo-rootwrap': + {'repository': 'git://git.openstack.org/openstack/oslo.rootwrap.git', + 'branch: 'master'}, + 'oslo-serialization': + {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git', + 'branch: 'master'}, + 'oslo-utils': + {'repository': 'git://git.openstack.org/openstack/oslo.utils.git', + 'branch: 'master'}, + 'pbr': + {'repository': 'git://git.openstack.org/openstack-dev/pbr.git', + 'branch: 'master'}, + 'python-keystoneclient': + {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git', + 'branch: 'master'}, + 'python-neutronclient': + {'repository': 'git://git.openstack.org/openstack/python-neutronclient.git', + 'branch: 'master'}, + 'python-novaclient': + {'repository': 'git://git.openstack.org/openstack/python-novaclient.git', + 'branch: 'master'}, + 'stevedore': + {'repository': 'git://git.openstack.org/openstack/stevedore.git', + 'branch: 'master'}, + 'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'master'}}" + # Restrictions This charm only support deployment with OpenStack Icehouse or better. diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 917cf211..714fd5d9 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 48b12f6f..aaa21c31 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/config.yaml b/config.yaml index aec778db..91c807cf 100644 --- a/config.yaml +++ b/config.yaml @@ -14,6 +14,25 @@ options: Note that updating this setting to a source that is known to provide a later version of OpenStack will trigger a software upgrade. + + Note that when openstack-origin-git is specified, openstack-specific + packages will be installed from source rather than from the + openstack-origin repository. + openstack-origin-git: + default: None + type: string + description: | + Specifies a YAML-formatted two-dimensional array listing the git + repositories and branches from which to install OpenStack and its + dependencies. + + Note that the installed config files will be determined based on + the OpenStack release of the openstack-origin option. + + Note also that this option is processed for the initial install + only. Setting this option after deployment is not supported. + + For more details see README.md. rabbit-user: default: neutron type: string diff --git a/hooks/neutron_api_hooks.py b/hooks/neutron_api_hooks.py index 63431968..aaeb7c4c 100755 --- a/hooks/neutron_api_hooks.py +++ b/hooks/neutron_api_hooks.py @@ -30,6 +30,7 @@ from charmhelpers.fetch import ( from charmhelpers.contrib.openstack.utils import ( configure_installation_source, + git_install_requested, openstack_upgrade_available, sync_db_with_multi_ipv6_addresses ) @@ -40,6 +41,7 @@ from neutron_api_utils import ( determine_packages, determine_ports, do_openstack_upgrade, + git_install, register_configs, restart_map, services, @@ -103,6 +105,14 @@ def install(): apt_update() apt_install(determine_packages(config('openstack-origin')), fatal=True) + + # NOTE(coreycb): This is temporary for sstack proxy, unless we decide + # we need to code proxy support into the charms. + os.environ["http_proxy"] = "http://squid.internal:3128" + os.environ["https_proxy"] = "https://squid.internal:3128" + + git_install(config('openstack-origin-git')) + [open_port(port) for port in determine_ports()] @@ -119,8 +129,9 @@ def config_changed(): config('database-user')) global CONFIGS - if openstack_upgrade_available('neutron-server'): - do_openstack_upgrade(CONFIGS) + if not git_install_requested(): + if openstack_upgrade_available('neutron-server'): + do_openstack_upgrade(CONFIGS) configure_https() update_nrpe_config() CONFIGS.write_all() @@ -135,6 +146,12 @@ def config_changed(): [cluster_joined(rid) for rid in relation_ids('cluster')] +#TODO(coreycb): For deploy from git support, need to implement action-set +# and action-get to trigger re-install of git-installed +# services. IIUC they'd be triggered via: +# juju do + + @hooks.hook('amqp-relation-joined') def amqp_joined(relation_id=None): relation_set(relation_id=relation_id, diff --git a/hooks/neutron_api_utils.py b/hooks/neutron_api_utils.py index 5d31a281..2ea595f6 100644 --- a/hooks/neutron_api_utils.py +++ b/hooks/neutron_api_utils.py @@ -1,6 +1,8 @@ from collections import OrderedDict from copy import deepcopy import os +import shutil +import yaml from base64 import b64encode from charmhelpers.contrib.openstack import context, templating from charmhelpers.contrib.openstack.neutron import ( @@ -10,6 +12,8 @@ from charmhelpers.contrib.openstack.neutron import ( from charmhelpers.contrib.openstack.utils import ( os_release, get_os_codename_install_source, + git_install_requested, + git_clone_and_install, configure_installation_source, ) @@ -26,7 +30,12 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.host import ( - lsb_release + adduser, + add_group, + add_user_to_group, + mkdir, + lsb_release, + write_file, ) import neutron_api_context @@ -52,6 +61,27 @@ KILO_PACKAGES = [ 'python-neutron-vpnaas', ] +BASE_GIT_PACKAGES = [ + 'libxml2-dev', + 'libxslt1-dev', + 'python-dev', + 'python-pip', + 'python-setuptools', + 'zlib1g-dev', +] + +# ubuntu packages that should not be installed when deploying from git +GIT_PACKAGE_BLACKLIST = [ + 'neutron-server', + 'python-keystoneclient', +] + +GIT_PACKAGE_BLACKLIST_KILO = [ + 'python-neutron-lbaas', + 'python-neutron-fwaas', + 'python-neutron-vpnaas', +] + BASE_SERVICES = [ 'neutron-server' ] @@ -110,14 +140,26 @@ def api_port(service): def determine_packages(source=None): # currently all packages match service names packages = [] + BASE_PACKAGES + for v in resource_map().values(): packages.extend(v['services']) pkgs = neutron_plugin_attribute(config('neutron-plugin'), 'server_packages', 'neutron') packages.extend(pkgs) + if get_os_codename_install_source(source) >= 'kilo': packages.extend(KILO_PACKAGES) + + if git_install_requested(): + packages.extend(BASE_GIT_PACKAGES) + # don't include packages that will be installed from git + for p in GIT_PACKAGE_BLACKLIST: + packages.remove(p) + if get_os_codename_install_source(source) >= 'kilo': + for p in GIT_PACKAGE_BLACKLIST_KILO: + packages.remove(p) + return list(set(packages)) @@ -242,3 +284,81 @@ def setup_ipv6(): ' main') apt_update() apt_install('haproxy/trusty-backports', fatal=True) + + +def git_install(projects): + """Perform setup, and install git repos specified in yaml parameter.""" + if git_install_requested(): + git_pre_install() + # NOTE(coreycb): charm-helpers needs support to take array of + # core_projects. That would allow all neutron* projects to be + # installed last. + core = ['neutron-fwaas', 'neutron-lbaas', 'neutron-vpnaas', 'neutron'] + git_clone_and_install(yaml.load(projects), core_projects=core) + git_post_install() + + +def git_pre_install(): + """Perform pre-install setup.""" + dirs = [ + '/etc/neutron', + '/etc/neutron/rootwrap.d', + '/etc/neutron/plugins', + '/var/lib/neutron', + '/var/lib/neutron/lock', + '/var/log/neutron', + ] + + logs = [ + '/var/log/neutron/server.log', + ] + + adduser('neutron', shell='/bin/bash', system_user=True) + add_group('neutron', system_group=True) + add_user_to_group('neutron', 'neutron') + + for d in dirs: + mkdir(d, owner='neutron', group='neutron', perms=0700, force=False) + + for l in logs: + write_file(l, '', owner='neutron', group='neutron', perms=0600) + + +def git_post_install(): + """Perform post-install setup.""" + src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/neutron-api.git/etc') + configs = { + 'api-paste': { + 'src': os.path.join(src_etc, 'api-paste.ini'), + 'dest': '/etc/neutron/api-paste.ini', + }, + 'debug-filters': { + 'src': os.path.join(src_etc, 'neutron/rootwrap.d/debug.filters'), + 'dest': '/etc/neutron/rootwrap.d/debug.filters', + }, + 'policy': { + 'src': os.path.join(src_etc, 'policy.json'), + 'dest': '/etc/neutron/policy.json', + }, + 'rootwrap': { + 'src': os.path.join(src_etc, 'rootwrap.conf'), + 'dest': '/etc/neutron/rootwrap.conf', + }, + } + + for conf, files in configs.iteritems(): + shutil.copyfile(files['src'], files['dest']) + + render('neutron-server.default', '/etc/default/neutron-server', {}, perms=0o440) + render('neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) + + neutron_api_context = { + 'service_description': 'Neutron API server', + 'charm_name': 'neutron-api', + 'process_name': 'neutron-server', + } + + render('upstart/neutron-server.upstart', '/etc/init/neutron.conf', + neutron_api_context, perms=0o644) + + service_start('neutron-server') diff --git a/templates/upstart/neutron-server.upstart b/templates/upstart/neutron-server.upstart new file mode 100644 index 00000000..2d3600e3 --- /dev/null +++ b/templates/upstart/neutron-server.upstart @@ -0,0 +1,22 @@ +description "{{ service_description }}" +author "Juju {{ charm_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +chdir /var/run + +pre-start script + mkdir -p /var/run/neutron + chown neutron:root /var/run/neutron +end script + +script + [ -r /etc/default/{{ process_name }} ] && . /etc/default/{{ process_name }} + [ -r "$NEUTRON_PLUGIN_CONFIG" ] && CONF_ARG="--config-file $NEUTRON_PLUGIN_CONFIG" + exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ + --config-file /etc/neutron/neutron.conf \ + --log-file /var/log/neutron/server.log $CONF_ARG +end script diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index e65a1481..9a2ca1be 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -65,11 +65,17 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): def _configure_services(self): """Configure all of the services.""" + # NOTE(coreycb): Added the following temporarily to test deploy from source + neutron_api_config = {'openstack-origin-git': + "{'neutron':" + " {'repository': 'git://git.openstack.org/openstack/neutron.git'," + " 'branch': 'stable/icehouse'}}"} keystone_config = {'admin-password': 'openstack', 'admin-token': 'ubuntutesting'} nova_cc_config = {'network-manager': 'Quantum', 'quantum-security-groups': 'yes'} - configs = {'keystone': keystone_config, + configs = {'neutron-api': neutron_api_config, + 'keystone': keystone_config, 'nova-cloud-controller': nova_cc_config} super(NeutronAPIBasicDeployment, self)._configure_services(configs)