From d64a586e9b2f166c92520d5811e28d3519cb34c3 Mon Sep 17 00:00:00 2001 From: Eran Rom Date: Mon, 31 Oct 2016 15:46:52 +0200 Subject: [PATCH] Using devstack for swift and keystone installation This change gets rid of our proprietary scripts for installing Swift and Keystone in favor of using devstack. So far we have used those scripts for: 1. The functional test gatejob 2. The s2aio.sh script The merits are: 1. No need to maintain these ourselves. Specifically, this gives us Linux distros other than Ubuntu for free 2. Being more Openstack compliant. Specifically, avoid packages conflicts that arise from our script usage of apt and the storlets setup environemnt which uses pip. The next step is to have a storlet devstack plugin that would allow to setup a dev/test env for storlets that is purely based on devstack. It might be that once having a full devstack integration we will clean the storlets repo from all ansible scripts, and drop the goal for supplying a multi-cluster installation support. Change-Id: I4731fe7369bc86bc626f08833aba08e5ff1d7184 --- doc/source/engine_dev_installation.rst | 172 +++++++----------- install/storlets/install_storlets.sh | 4 +- install/storlets/installation_vars.yml-sample | 8 +- install/storlets/prepare_storlets_install.sh | 5 +- .../common_templates/swift_middleware_conf | 4 +- .../tasks/main.yml | 6 +- .../deploy_default_tenant/tasks/main.yml | 2 +- .../tasks/ubuntu_14.04_jre8.yml | 1 + .../tasks/main.yml | 21 +-- .../host_storlet_engine/handlers/main.yml | 2 +- .../roles/host_storlet_engine/tasks/main.yml | 2 +- .../tasks/main.yml | 2 +- .../handlers/main.yml | 2 +- .../tasks/main.yml | 4 + .../tasks/main.yml | 3 +- .../roles/reconfigure_swift/tasks/main.yml | 2 +- install/storlets/templates/cluster_config | 7 +- install/swift/README.md | 7 - install/swift/install_swift.sh | 119 ++++++------ install/swift/localrc.sample | 33 ++++ install/swift/prepare_swift_install.yml | 41 ----- .../roles/prepare_loop_device/tasks/main.yml | 31 ---- .../roles/pull_swift_ansible/tasks/main.yml | 38 ---- install/swift/templates/cluster_config | 83 --------- install/swift/vars.yml-sample | 22 --- s2aio.sh | 21 ++- 26 files changed, 213 insertions(+), 429 deletions(-) delete mode 100644 install/swift/README.md create mode 100644 install/swift/localrc.sample delete mode 100644 install/swift/prepare_swift_install.yml delete mode 100644 install/swift/roles/prepare_loop_device/tasks/main.yml delete mode 100644 install/swift/roles/pull_swift_ansible/tasks/main.yml delete mode 100644 install/swift/templates/cluster_config delete mode 100644 install/swift/vars.yml-sample diff --git a/doc/source/engine_dev_installation.rst b/doc/source/engine_dev_installation.rst index 16039421..d2ce8134 100644 --- a/doc/source/engine_dev_installation.rst +++ b/doc/source/engine_dev_installation.rst @@ -2,123 +2,93 @@ Installing a Development Environment ==================================== This guide gives a step by step installation instructions that are simpler then what the s2aio.sh script does (e.g. it does not involve a docker registry -installation and configuration). Performing those instructions, as oppose to -just running s2aio.sh, can help in better inderstanding the overall system. +installation and configuration). -The guide assumes that you already have installed SAIO as described -in http://docs.openstack.org/developer/swift/development_saio.html -It further assumes that you used a partition for storage that is -mounted on /mnt/sdb1, and that the proxy port is 8080. +The below steps must be executed using a passwordless sudoer user. -.. note:: +Install Swift and Keystone using devstack +----------------------------------------- - This guide assumes that the user executing these instructions - is the same user who installed SAIO. Specifically, it assumes - that $USER would evaluate to the same user who instslled SAIO. - -This process has been tested on Ubuntu 14.04 using Swift 2.7.0. - -.. note:: - - Completing the SAIO installation on Ubuntu 14.04 requires an newer version of pip, setuptools and pbr. - To upgrade pip see - http://unix.stackexchange.com/questions/36710/how-can-i-upgrade-pip-on-ubuntu-10-04 - To upgrade setuptools and pbr just do: - pip install --upgrade pbr - pip install --upgrade setuptools - -Make Swift use Keystone -======================= - -SAIO uses tmpauth as an auth middleware. While storlets do not have a -direct dependency on the auth middleware used, Keystone seems to be -the de-facto standard in deployments, and so we use it. - -Keystone Installation ---------------------- -Below we use the ubuntu mitaka cloud archive. -As a rule of thumb, use the archive closest to the -installed SAIO version +Clone devstack: :: - sudo apt-get install --upgrade software-properties-common - sudo add-apt-repository cloud-archive:mitaka - sudo apt-get update - sudo apt-get install keystone - sudo sed -i 's/#admin_token = /admin_token = ADMIN/g' /etc/keystone/keystone.conf - sudo service keystone restart + git clone git://github.com/openstack-dev/devstack.git -To configure Keystone you would also need to: +Create a localrc file under the devstack repository root directory: :: - sudo apt-get install python-openstackclient + ENABLE_HTTPD_MOD_WSGI_SERVICES=False + KEYSTONE_IP=127.0.0.1 + SWIFT_IP=127.0.0.1 + ENABLED_SERVICES=key,swift,mysql + ADMIN_USER=admin + ADMIN_PASSWORD=$ADMIN_USER + ADMIN_PROJECT=ADMIN_USER + DATABASE_PASSWORD=admin + RABBIT_PASSWORD=$ADMIN_PASSWORD + SERVICE_PASSWORD=$ADMIN_PASSWORD + OS_IDENTITY_API_VERSION=2 + OS_AUTH_URL="http://$KEYSTONE_IP:5000/v2.0" + OS_USERNAME=$ADMIN_USER + OS_USER_DOMAIN_ID=default + OS_PASSWORD=$ADMIN_PASSWORD + OS_PROJECT_NAME=$ADMIN_USER + OS_PROJECT_DOMAIN_ID=default + OS_REGION_NAME=RegionOne -Initial Keystone Configutation ------------------------------- -The following creates the identity and object store service together with their endpoints. -Note the usage of port 8080 in the object store public url. If your SAIO uses another port -change the below command accordingly. + SERVICE_HOST=$SWIFT_IP + SWIFT_SERVICE_PROTOCOL=http + SWIFT_DEFAULT_BIND_PORT=8080 + SWIFT_SERVICE_LOCAL_HOST=$SERVICE_HOST + SWIFT_SERVICE_LISTEN_ADDRESS=$SERVICE_HOST + # Use minimum 2GB for running the storlets tests + SWIFT_LOOPBACK_DISK_SIZE=20G + SWIFT_HASH=1234567890 + +Run the stack.sh script. +Before proceeding, we need to stop the +swift instances that were executed by the +stack.sh. From the same directory do: + +:: + source functions + source lib/swift + stop_swift + +Finally, add the swift devices to fstab: :: - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ service create identity - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ endpoint create --publicurl http://127.0.0.1:5000/v2.0 --adminurl http://127.0.0.1:35357/v2.0 identity - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ service create object-store - openstack --os-url http://127.0.0.1:35357/v2.0/ --os-token ADMIN endpoint create --publicurl 'http://127.0.0.1:8080/v1/AUTH_$(tenant_id)s' object-store + sudo sh -c 'echo "/opt/stack/data/swift/drives/images/swift.img /opt/stack/data/swift/drives/sdb1 xfs loop" >> /etc/fstab' -Create a Swift admin user. The Swift proxy will use this user to authorize tokens with Keystone +Configure a user and project in Keystone +---------------------------------------- + +We use the opnstack cli to configure a user and project +used by the storlets functional tests. We start by +defining some environment variables: :: - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ role create admin - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ project create service - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ user create swift --password passw0rd - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ role add --user swift --project service admin + export OS_IDENTITY_API_VERSION=2 + export OS_AUTH_URL="http://$KEYSTONE_IP:5000/v2.0" + export OS_USERNAME=$ADMIN_USER + export OS_USER_DOMAIN_ID=default + export OS_PASSWORD=$ADMIN_PASSWORD + export OS_PROJECT_NAME=$ADMIN_USER + export OS_PROJECT_DOMAIN_ID=default + export OS_REGION_NAME=RegionOne -Create a Swift end user that is admin. The admin role is necessary as we want a user that can create containers and set account metadata +We now create the project and user: :: - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ project create test - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ user create tester --password testing - openstack --os-token ADMIN --os-url http://127.0.0.1:35357/v2.0/ role add --user tester --project test admin - -Configure Swift to work with Keystone -------------------------------------- -Edit the file /etc/swift/proxy-server.conf as follows: - -1. Use keystone instead of tmpauth - -:: - - sudo sed -i '0,/tempauth/{s/tempauth/authtoken keystoneauth/}' /etc/swift/proxy-server.conf - -2. Add the following blocks at the end of /etc/swift/proxy-server.conf - -:: - - [filter:authtoken] - paste.filter_factory = keystonemiddleware.auth_token:filter_factory - auth_url=http://127.0.0.1:35357 - auth_type=password - insecure=true - project_name=service - username=swift - password=passw0rd - delay_auth_decision = True - - [filter:keystoneauth] - use = egg:swift#keystoneauth - -Restart the proxy server - -:: - - sudo swift-init proxy-server restart - + openstack project create test + openstack user create --project test --password testing tester + openstack role add --user tester --project test admin We now test that the setup by having the user 'tester' to stat the account 'test'. We use the Swift client cli. A convenient way to do so is to edit the user's .bashrc adding the lines: @@ -526,26 +496,18 @@ content. :: { - "groups" : { - "storlet-mgmt": [ "127.0.0.1" ], - "storlet-proxy": [ "127.0.0.1" ], - "storlet-storage": [ "127.0.0.1" ], - "docker": [ "127.0.0.1" ] - }, "all" : { "docker_device": "/home/docker_device", "storlet_source_dir": "~/storlets", - "python_dist_packages_dir": "usr/local/lib/python2.7/dist-packages", - "storlet_gateway_conf_file": "/etc/swift/storlet_docker_gateway.conf", - "keystone_endpoint_host": "127.0.0.1", - "keystone_admin_url": "http://127.0.0.1:35357/v2.0", "keystone_public_url": "http://127.0.0.1:5000/v2.0", "swift_endpoint_host": "127.0.0.1", "swift_public_url": "http://127.0.0.1:8080/v1", "storlets_enabled_attribute_name": "storlet-enabled", "storlets_default_tenant_name": "test", "storlets_default_tenant_user_name": "tester", - "storlets_default_tenant_user_password": "testing" + "storlets_default_tenant_user_password": "testing", + "storlets_default_tenant_member_user": "tester_member", + "storlets_default_tenant_member_password": "member", } } diff --git a/install/storlets/install_storlets.sh b/install/storlets/install_storlets.sh index ae6ae607..ac52f02b 100755 --- a/install/storlets/install_storlets.sh +++ b/install/storlets/install_storlets.sh @@ -8,10 +8,10 @@ echo "Running hosts docker_cluster playbook" ansible-playbook -s -i storlets_dynamic_inventory.py docker_cluster.yml echo "Running the host_side_storlet_engine playbook" -ansible-playbook -s -i storlets_dynamic_inventory.py host_side_storlet_engine.yml +ansible-playbook -i storlets_dynamic_inventory.py host_side_storlet_engine.yml echo "Running the enable_storlets_for_default_tenant playbook" -ansible-playbook -s -i storlets_dynamic_inventory.py enable_storlets_for_default_tenant.yml +ansible-playbook -i storlets_dynamic_inventory.py enable_storlets_for_default_tenant.yml # TODO(eranr): Get back to the ant dev playbooks!!! set +eu diff --git a/install/storlets/installation_vars.yml-sample b/install/storlets/installation_vars.yml-sample index 91b3643e..d895c31e 100644 --- a/install/storlets/installation_vars.yml-sample +++ b/install/storlets/installation_vars.yml-sample @@ -26,10 +26,12 @@ storlets_default_tenant_name: "test" storlets_default_tenant_user_name: "tester" storlets_default_tenant_user_password: "testing" keystone_endpoint_host: 127.0.0.1 -keystone_admin_token: ADMIN -keystone_admin_password: passw0rd +keystone_admin_user: admin +keystone_admin_password: admin +keystone_admin_project: admin swift_endpoint_host: 127.0.0.1 -swift_endpoint_port: 80 +swift_endpoint_port: 8080 swift_run_time_user: swift_run_time_group: +swift_run_time_dir: docker_device: /home/docker_device diff --git a/install/storlets/prepare_storlets_install.sh b/install/storlets/prepare_storlets_install.sh index 10262ae9..0a3dde16 100755 --- a/install/storlets/prepare_storlets_install.sh +++ b/install/storlets/prepare_storlets_install.sh @@ -66,8 +66,9 @@ else sed -i 's//root/g' deploy/installation_vars.yml sed -i 's/127.0.0.1/'$SWIFT_IP'/g' deploy/installation_vars.yml fi - sed -i 's//swift/g' deploy/installation_vars.yml - sed -i 's//swift/g' deploy/installation_vars.yml + sed -i 's//'$USER'/g' deploy/installation_vars.yml + sed -i 's//'$USER'/g' deploy/installation_vars.yml + sed -i 's//\/opt\/stack\/data\/swift\/run/g' deploy/installation_vars.yml if [ "$FLAVOR" == "jenkins" ]; then sed -i 's//\/home\/'$USER'\/workspace\/gate-storlets-functional\//g' deploy/installation_vars.yml else diff --git a/install/storlets/roles/common_templates/swift_middleware_conf b/install/storlets/roles/common_templates/swift_middleware_conf index e7435370..5fdbb3bb 100644 --- a/install/storlets/roles/common_templates/swift_middleware_conf +++ b/install/storlets/roles/common_templates/swift_middleware_conf @@ -3,8 +3,8 @@ proxy_server_conf_file = /etc/swift/proxy-server.conf storlet_proxy_server_conf_file = /etc/swift/storlet-proxy-server.conf [object-confs] -#object_server_conf_files = /etc/swift/object-server/1.conf, /etc/swift/object-server/2.conf, /etc/swift/object-server/3.conf, /etc/swift/object-server/4.conf -object_server_conf_files = /etc/swift/object-server.conf +object_server_conf_files = /etc/swift/object-server/1.conf, /etc/swift/object-server/2.conf, /etc/swift/object-server/3.conf, /etc/swift/object-server/4.conf +#object_server_conf_files = /etc/swift/object-server.conf [common-confs] storlet_middleware = {{ storlet_middleware }} diff --git a/install/storlets/roles/create_default_tenant_docker_image/tasks/main.yml b/install/storlets/roles/create_default_tenant_docker_image/tasks/main.yml index 6d8aac9d..19cf6c6f 100644 --- a/install/storlets/roles/create_default_tenant_docker_image/tasks/main.yml +++ b/install/storlets/roles/create_default_tenant_docker_image/tasks/main.yml @@ -15,7 +15,7 @@ # TODO(eranr): Change the below to use Ansible os_project_facts - name: Get the tenant id from Keystone - shell: openstack --os-url={{ keystone_admin_url }} --os-token={{ keystone_admin_token }} project list | grep {{ tenant_name }} | awk '{print $2}' + shell: openstack --os-auth-url={{ keystone_public_url }} --os-username={{ keystone_admin_user }} --os-password={{ keystone_admin_password }} --os-project-name={{ keystone_admin_project }} project list | grep {{ tenant_name }} | awk '{print $2}' register: tenant_id - name: create the tenant specific docker image step 1 - create repo dir @@ -24,6 +24,7 @@ dest: "/data/registry/repositories/{{ tenant_id.stdout_lines[0] }}" owner: root mode: 0755 + become: true - name: create the tenant specific docker image step 2 - create Docker file template: @@ -31,13 +32,16 @@ dest: "/data/registry/repositories/{{ tenant_id.stdout_lines[0] }}/Dockerfile" owner: root mode: 0755 + become: true - name: "Build the image {{ tenant_id.stdout_lines[0] }}" command: "docker build -q -t {{ hostvars[groups['docker'][0]]['inventory_hostname'] }}:{{ docker_registry_port }}/{{ tenant_id.stdout_lines[0] }} ." args: chdir: "/data/registry/repositories/{{ tenant_id.stdout_lines[0] }}" register: command_result + become: true failed_when: "'sha256:' not in command_result.stdout" - name: Push the image to the global registry command: "docker push {{ hostvars[groups['docker'][0]]['inventory_hostname'] }}:{{ docker_registry_port }}/{{ tenant_id.stdout_lines[0] }}" + become: true diff --git a/install/storlets/roles/deploy_default_tenant/tasks/main.yml b/install/storlets/roles/deploy_default_tenant/tasks/main.yml index 8b0b29b9..f6489122 100644 --- a/install/storlets/roles/deploy_default_tenant/tasks/main.yml +++ b/install/storlets/roles/deploy_default_tenant/tasks/main.yml @@ -14,7 +14,7 @@ #--------------------------------------------------------------------------- # TODO(eranr): Change thebelow to use Ansible os_project_facts module - name: Get the defaut tenant id from Keystone - shell: openstack --os-url={{ keystone_admin_url }} --os-token={{ keystone_admin_token }} project list | grep {{ storlets_default_tenant_name }} | awk '{print $2}' + shell: openstack --os-auth-url={{ keystone_public_url }} --os-username= {{ keystone_admin_user }} --os-password={{ keystone_admin_password }} --os-project-name={{ keystone_admin_project }} project list | grep {{ storlets_default_tenant_name }} | awk '{print $2}' register: tenant_id - name: Print the tenant id whose container we are processing diff --git a/install/storlets/roles/docker_base_jre_image/tasks/ubuntu_14.04_jre8.yml b/install/storlets/roles/docker_base_jre_image/tasks/ubuntu_14.04_jre8.yml index fe8cd3de..a44f69ba 100644 --- a/install/storlets/roles/docker_base_jre_image/tasks/ubuntu_14.04_jre8.yml +++ b/install/storlets/roles/docker_base_jre_image/tasks/ubuntu_14.04_jre8.yml @@ -38,6 +38,7 @@ get_url: url: "{{ item }}" dest: /data/registry/repositories/ubuntu_14.04_jre8 + retries: 3 with_items: - http://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json_simple-1.1.jar - http://www.slf4j.org/dist/slf4j-1.7.7.tar.gz diff --git a/install/storlets/roles/enable_default_swift_account_for_storlets/tasks/main.yml b/install/storlets/roles/enable_default_swift_account_for_storlets/tasks/main.yml index 10dd3e15..a4a6c103 100644 --- a/install/storlets/roles/enable_default_swift_account_for_storlets/tasks/main.yml +++ b/install/storlets/roles/enable_default_swift_account_for_storlets/tasks/main.yml @@ -13,15 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO(eranr): Remove this once we move to work with Ansible os_user, os_project_facts -- name: Install dependencies - apt: - pkg: "{{ item }}" - state: present - with_items: - - python-swiftclient - - python-openstackclient - # TODO(eran): Get the enabled flag name from config file - name: Set account metadata in swift -- enable storlets command: swift post @@ -72,20 +63,12 @@ --write-acl "{{ storlet_management_account }}:{{ storlet_management_admin_username }}" docker_images -# TODO(eranr): Use ansible os_user instead of openstackclient. -# The below seems to have a problem with the auth parameters. -# os_user: -# auth_type: v2token -# auth: url="{{ keystone_admin_url }}" token="{{ keystone_admin_token }}" tenant_name="{{ storlets_default_tenant_name }}" -# name: "{{ storlets_default_tenant_member_user }}" -# password: "{{ storlets_default_tenant_member_password }}" - - name: test if need to create member user - shell: openstack --os-url={{ keystone_admin_url }} --os-token={{ keystone_admin_token }} user list --project "{{ storlets_default_tenant_name }}" | grep "{{ storlets_default_tenant_member_user }}" | wc -l + shell: openstack --os-auth-url={{ keystone_public_url }} --os-username={{ keystone_admin_user }} --os-password={{ keystone_admin_password }} --os-project-name={{ keystone_admin_project }} user list --project "{{ storlets_default_tenant_name }}" | grep "{{ storlets_default_tenant_member_user }}" | wc -l register: member_user - debug: msg="{{ member_user }}" - name: Create member user - shell: openstack --os-url={{ keystone_admin_url }} --os-token={{ keystone_admin_token }} user create --project "{{ storlets_default_tenant_name }}" --password "{{ storlets_default_tenant_member_password }}" --enable "{{ storlets_default_tenant_member_user }}" + shell: openstack --os-auth-url={{ keystone_public_url }} --os-username={{ keystone_admin_user }} --os-password={{ keystone_admin_password }} --os-project-name={{ keystone_admin_project }} user create --project "{{ storlets_default_tenant_name }}" --password "{{ storlets_default_tenant_member_password }}" --enable "{{ storlets_default_tenant_member_user }}" when: "'0' in member_user.stdout" diff --git a/install/storlets/roles/host_storlet_engine/handlers/main.yml b/install/storlets/roles/host_storlet_engine/handlers/main.yml index 1bf655e6..199ff3c3 100644 --- a/install/storlets/roles/host_storlet_engine/handlers/main.yml +++ b/install/storlets/roles/host_storlet_engine/handlers/main.yml @@ -14,7 +14,7 @@ #--------------------------------------------------------------------------- - name: restart swift - shell: swift-init all {{ item }} + shell: swift-init --run-dir="{{swift_run_time_dir}}" all {{ item }} with_items: - stop - start diff --git a/install/storlets/roles/host_storlet_engine/tasks/main.yml b/install/storlets/roles/host_storlet_engine/tasks/main.yml index 2bb6be8e..3be2dbff 100644 --- a/install/storlets/roles/host_storlet_engine/tasks/main.yml +++ b/install/storlets/roles/host_storlet_engine/tasks/main.yml @@ -121,5 +121,5 @@ python swift_config.py install swift_middleware_conf "{{ swift_run_time_user }}" - name: restart swift - shell: swift-init all restart + shell: swift-init --run-dir="{{swift_run_time_dir}}" all restart ignore_errors: yes diff --git a/install/storlets/roles/host_storlet_engine_configure/tasks/main.yml b/install/storlets/roles/host_storlet_engine_configure/tasks/main.yml index d6bb013f..ea0b8c2a 100644 --- a/install/storlets/roles/host_storlet_engine_configure/tasks/main.yml +++ b/install/storlets/roles/host_storlet_engine_configure/tasks/main.yml @@ -42,5 +42,5 @@ python swift_config.py install swift_middleware_conf "{{ swift_run_time_user }}" - name: restart swift - shell: swift-init all restart + shell: swift-init --run-dir="{{swift_run_time_dir}}" all restart ignore_errors: yes diff --git a/install/storlets/roles/host_storlet_engine_install/handlers/main.yml b/install/storlets/roles/host_storlet_engine_install/handlers/main.yml index 1bf655e6..199ff3c3 100644 --- a/install/storlets/roles/host_storlet_engine_install/handlers/main.yml +++ b/install/storlets/roles/host_storlet_engine_install/handlers/main.yml @@ -14,7 +14,7 @@ #--------------------------------------------------------------------------- - name: restart swift - shell: swift-init all {{ item }} + shell: swift-init --run-dir="{{swift_run_time_dir}}" all {{ item }} with_items: - stop - start diff --git a/install/storlets/roles/host_storlet_engine_install/tasks/main.yml b/install/storlets/roles/host_storlet_engine_install/tasks/main.yml index 40a24e5c..796c6868 100644 --- a/install/storlets/roles/host_storlet_engine_install/tasks/main.yml +++ b/install/storlets/roles/host_storlet_engine_install/tasks/main.yml @@ -15,12 +15,15 @@ - name: install python facade on host unarchive: owner=root group=root mode=0755 dest=/ src={{ SBusPythonFacade_dir }}/dist/{{ SBusPythonFacade_egg_dist }} + become: true - name: install sbus transport layer on host copy: src={{ SBusTransportLayer_dir }}/{{ sbus_so }} dest=/{{python_dist_packages_dir}} owner=root group=root mode=0755 + become: true - name: install swift middleware on host unarchive: owner=root group=root mode=0755 dest=/ src={{ swift_middleware_dir }}/dist/{{ swift_middleware_egg_dist }} + become: true - stat: path={{ docker_device }}/scripts register: st @@ -38,3 +41,4 @@ with_items: - "Engine/SMScripts/bin/restart_docker_container" - "Engine/SMScripts/send_halt_cmd_to_daemon_factory.py" + become: true diff --git a/install/storlets/roles/pull_default_tenant_docker_image/tasks/main.yml b/install/storlets/roles/pull_default_tenant_docker_image/tasks/main.yml index a5558a35..040335b5 100644 --- a/install/storlets/roles/pull_default_tenant_docker_image/tasks/main.yml +++ b/install/storlets/roles/pull_default_tenant_docker_image/tasks/main.yml @@ -15,11 +15,12 @@ # TODO(eranr): Chaneg the below to use Ansible os_project_facts - name: Get the tenant id from Keystone - shell: openstack --os-url={{ keystone_admin_url }} --os-token={{ keystone_admin_token }} project list | grep {{ tenant_name }} | awk '{print $2}' + shell: openstack --os-auth-url={{keystone_public_url}} --os-username={{keystone_admin_user}} --os-password={{keystone_admin_password}} --os-project-name={{keystone_admin_project}} project list | grep {{ tenant_name }} | awk '{print $2}' register: tenant_id - name: docker pull command: docker pull {{ hostvars[groups['docker'][0]]['inventory_hostname'] }}:{{ docker_registry_port }}/{{ tenant_id.stdout_lines[0] }} + become: true - name: shutdown_container shell: "/usr/bin/python {{ docker_device }}/scripts/send_halt_cmd_to_daemon_factory.py {{ docker_device }}/pipes/scopes/AUTH_{{ tenant_id.stdout_lines[0] }}/factory_pipe" diff --git a/install/storlets/roles/reconfigure_swift/tasks/main.yml b/install/storlets/roles/reconfigure_swift/tasks/main.yml index 63b8a69d..788db1d2 100644 --- a/install/storlets/roles/reconfigure_swift/tasks/main.yml +++ b/install/storlets/roles/reconfigure_swift/tasks/main.yml @@ -33,4 +33,4 @@ python swift_config.py remove swift_middleware_conf "{{ swift_run_time_user }}" - name: restart swift - shell: swift-init all restart + shell: swift-init --run-dir="{{swift_run_time_dir}}" all restart diff --git a/install/storlets/templates/cluster_config b/install/storlets/templates/cluster_config index b13d99df..aea9b62e 100644 --- a/install/storlets/templates/cluster_config +++ b/install/storlets/templates/cluster_config @@ -12,13 +12,14 @@ "python_dist_packages_dir": "usr/local/lib/python2.7/dist-packages", "storlet_gateway_conf_file": "/etc/swift/storlet_docker_gateway.conf", "keystone_endpoint_host": "{{ keystone_endpoint_host }}", - "keystone_admin_url": "http://{{ keystone_endpoint_host }}:35357/v2.0", "keystone_public_url": "http://{{ keystone_endpoint_host }}:5000/v2.0", - "keystone_admin_token": "{{ keystone_admin_token }}", + "keystone_admin_password": "{{ keystone_admin_password }}", + "keystone_admin_user": "{{keystone_admin_user}}", + "keystone_admin_project": "{{keystone_admin_project}}", "swift_endpoint_host": "{{ swift_endpoint_host }}", - "swift_public_url": "http://{{ swift_endpoint_host }}:{{ swift_endpoint_port }}/v1", "swift_run_time_user" : "{{ swift_run_time_user }}", "swift_run_time_group" : "{{ swift_run_time_group }}", + "swift_run_time_dir": "{{ swift_run_time_dir }}", "storlets_management_user": "{{ storlets_management_user }}", "storlet_management_account": "{{ storlet_management_account }}", "storlet_management_admin_username": "{{ storlet_management_admin_username }}", diff --git a/install/swift/README.md b/install/swift/README.md deleted file mode 100644 index 952eb63b..00000000 --- a/install/swift/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Installation process: -1. Decide on swift device: either loop or real. Once decided - set vars.yml-sample accordingly and copy to vars.yml -2. Prepare the loop device if this is the choice -3. Create the cluster_config file that describes the cluster based on a template. -4. pull the Swift ansible installation script -5. Copy cluster_config to the proper location of the Swift ansible installation script -6. Invoke swift installation: cd to repo/provisioniing, ansible-playbook tralala diff --git a/install/swift/install_swift.sh b/install/swift/install_swift.sh index 550951e4..1eaf01c8 100755 --- a/install/swift/install_swift.sh +++ b/install/swift/install_swift.sh @@ -1,22 +1,13 @@ #!/bin/bash set -eu -# Invokes the Swift install process that is based on -# https://github.com/Open-I-Beam/swift-install -# with appropriate pre install preparations -# This is a dev oriented Swift installation that -# uses Keystone and a single device for all rings. -# TODO: Move swift ansible scripts pull from here -# to the swift-install module +# Invokes a devstack install that consists of +# keyastone and swift. -# The script takes a block device name as an optional parameter -# The device name can be either 'loop0' or any block device under /dev -# that can be formatted and mounted as a Swift device. -# The script assume it 'can sudo' - -if [ "$#" -ne 3 ]; then - echo "Usage: $0 [target] [ip] [device-name]" +if [ "$#" -ne 2 ] && [ "$#" -ne 3 ]; then + echo "Usage: $0 [target] [ip] [flavour]" echo "target = host | docker" + echo "optionally specify flavour=dev installation" exit fi @@ -27,50 +18,68 @@ if [ "$TARGET" != "host" ] && [ "$TARGET" != "docker" ]; then fi SWIFT_IP=$2 -DEVICE=$3 -if [ $DEVICE != 'loop0' ] && [ ! -b "/dev/$DEVICE" ]; then - echo "$DEVICE is not a block device" - exit -fi -REPODIR='/tmp' -REPODIR_REPLACE='\/tmp' - -echo "$DEVICE will be used as a block device for Swift" -if [ ! -e vars.yml ]; then - cp vars.yml-sample vars.yml - sudo sed -i 's//'$DEVICE'/g' vars.yml - sudo sed -i 's//'$REPODIR_REPLACE'/g' vars.yml - sudo sed -i 's//'$SWIFT_IP'/g' vars.yml -fi - -if [ $TARGET == 'docker' ]; then - cat > hosts <> ~/.ssh/authorized_keys' +if [ "$#" -eq 3 ]; then + FLAVOR=$3 else - cat > hosts </'$SWIFT_IP'/g' $DEVSTACK_DIR/localrc + if [ $FLAVOR = 'dev' ]; then + sed -i 's//admin/g' $DEVSTACK_DIR/localrc + else + sed -i '//d' $DEVSTACK_DIR/localrc + fi +fi + +# run devstack +cd $DEVSTACK_DIR + +# This is an ugly hack to overcome +# devstack installation problem in docker +# TODO(eranr): address this after +# adding a devstack plugin to storlets! +if [ "$TARGET" == "docker" ]; then + set +e + ./stack.sh + sudo service mysql start + set -e +fi +./stack.sh +# stack.sh starts swift in a non-standard manner +# we thus stop it before continuing +set +u +source functions +source lib/swift +stop_swift +set -u +cd - + +# add tester, testing, test which is admin +source $DEVSTACK_DIR/localrc +project_test_created=$(openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | wc -l) +if [ $project_test_created -eq 0 ]; then + openstack project create $SWIFT_DEFAULT_PROJECT +fi +user_tester_created=$(openstack user list | grep -w $SWIFT_DEFAULT_USER | wc -l) +if [ $user_tester_created -eq 0 ]; then + openstack user create --project $SWIFT_DEFAULT_PROJECT --password $SWIFT_DEFAULT_USER_PWD $SWIFT_DEFAULT_USER + openstack role add --user $SWIFT_DEFAULT_USER --project $SWIFT_DEFAULT_PROJECT admin +fi + +# add entry to fstab +mount_added=$(grep swift.img /etc/fstab | wc -l) +if [ $mount_added -eq 0 ]; then + sudo sh -c 'echo "/opt/stack/data/swift/drives/images/swift.img /opt/stack/data/swift/drives/sdb1 xfs loop" >> /etc/fstab' +fi set +eu -# NOTE: Right now, swift-install/provisioning has some tasks to kill no -# running processes (e.g. swift-init proxy stop for clean environment) and -# it will make a non zero exit code causes gate failure so remove set -eu -# trusting those script. (Hopefully, it could be solved in the script) -if [ $TARGET == 'host' ]; then - cd $REPODIR/swift-install/provisioning - ansible-playbook -s -i swift_dynamic_inventory.py main-install.yml -else - ssh root@$SWIFT_IP "bash -c 'cd /tmp/swift-install/provisioning ; ansible-playbook -s -i swift_dynamic_inventory.py main-install.yml'" -fi diff --git a/install/swift/localrc.sample b/install/swift/localrc.sample new file mode 100644 index 00000000..959b33aa --- /dev/null +++ b/install/swift/localrc.sample @@ -0,0 +1,33 @@ +ENABLE_HTTPD_MOD_WSGI_SERVICES=False +KEYSTONE_IP= +SWIFT_IP= +ENABLED_SERVICES=key,swift,mysql +ADMIN_USER=admin +ADMIN_PASSWORD=admin +ADMIN_PROJECT=admin +DATABASE_PASSWORD= +RABBIT_PASSWORD=$ADMIN_PASSWORD +SERVICE_PASSWORD=$ADMIN_PASSWORD +export OS_IDENTITY_API_VERSION=2 +export OS_AUTH_URL="http://$KEYSTONE_IP:5000/v2.0" +export OS_USERNAME=$ADMIN_USER +export OS_USER_DOMAIN_ID=default +export OS_PASSWORD=$ADMIN_PASSWORD +export OS_PROJECT_NAME=$ADMIN_USER +export OS_PROJECT_DOMAIN_ID=default +export OS_REGION_NAME=RegionOne + +HOST_IP=$SWIFT_IP +SWIFT_SERVICE_PROTOCOL=http +SWIFT_DEFAULT_BIND_PORT=8080 +# service local host is used for ring building +SWIFT_SERVICE_LOCAL_HOST=$HOST_IP +# service listen address for prox +SWIFT_SERVICE_LISTEN_ADDRESS=$HOST_IP +SWIFT_LOOPBACK_DISK_SIZE=20G +SWIFT_MAX_FILE_SIZE=5368709122 +SWIFT_HASH=1234567890 + +SWIFT_DEFAULT_PROJECT=test +SWIFT_DEFAULT_USER=tester +SWIFT_DEFAULT_USER_PWD=testing diff --git a/install/swift/prepare_swift_install.yml b/install/swift/prepare_swift_install.yml deleted file mode 100644 index 089c2504..00000000 --- a/install/swift/prepare_swift_install.yml +++ /dev/null @@ -1,41 +0,0 @@ -#--------------------------------------------------------------------------- -# Copyright IBM Corp. 2015, 2016 All Rights Reserved -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# Limitations under the License. -#--------------------------------------------------------------------------- - -- name: prepare loop device - hosts: s2aio - vars_files: - - [vars.yml] - roles: - - {role: prepare_loop_device, when: "swift_device == 'loop0'"} - -- name: cluster_config.json from template - hosts: s2aio - vars_files: - - [vars.yml] - tasks: - - stat: path="/tmp/cluster_config.json" - register: st0 - - - template: - src: templates/cluster_config - dest: "/tmp/cluster_config.json" - when: not st0.stat.exists - -- name: pull Swift installation scripts - hosts: s2aio - vars_files: - - [vars.yml] - roles: - - role: pull_swift_ansible diff --git a/install/swift/roles/prepare_loop_device/tasks/main.yml b/install/swift/roles/prepare_loop_device/tasks/main.yml deleted file mode 100644 index bd1bbaca..00000000 --- a/install/swift/roles/prepare_loop_device/tasks/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -#--------------------------------------------------------------------------- -# Copyright IBM Corp. 2015, 2015 All Rights Reserved -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# Limitations under the License. -#--------------------------------------------------------------------------- - -- name: mkdir srv directory - become: yes - become_user: root - file: path=/srv state=directory - ignore_errors: yes - -- name: create disk file - become: yes - become_user: root - command: truncate -s 10GB /srv/swift-disk - -- name: losetup - become: yes - become_user: root - command: losetup /dev/loop0 /srv/swift-disk - ignore_errors: yes diff --git a/install/swift/roles/pull_swift_ansible/tasks/main.yml b/install/swift/roles/pull_swift_ansible/tasks/main.yml deleted file mode 100644 index 01ae6387..00000000 --- a/install/swift/roles/pull_swift_ansible/tasks/main.yml +++ /dev/null @@ -1,38 +0,0 @@ -#--------------------------------------------------------------------------- -# Copyright IBM Corp. 2015, 2015 All Rights Reserved -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# Limitations under the License. -#--------------------------------------------------------------------------- - -- stat: path={{ swift_install_reop_dir }} - register: inst_dir -- file: path={{ swift_install_reop_dir }} state=directory recurse=yes - when: not inst_dir.stat.exists - -- name: clone swift_install repo - stat: path="{{ swift_install_reop_dir }}/{{swift_install_repo_name}}" - register: repo_dir - -- git: repo={{ swift_install_repo_url }} - dest={{ swift_install_reop_dir }}/{{ swift_install_repo_name }} - version={{ swift_install_repo_branch }} - when: not repo_dir.stat.exists - -- name: clone openstack ansible libraries (for keystone configuration) - git: repo=https://github.com/openstack-ansible/openstack-ansible-modules.git - dest={{ swift_install_reop_dir }}/{{ swift_install_repo_name }}/provisioning/openstack-ansible-modules - version="master" - -- name: Copy cluster_config.json to the swift install location - copy: src=/tmp/cluster_config.json dest="{{ swift_install_reop_dir }}/{{ swift_install_repo_name }}/provisioning/cluster_config.json" remote_src=yes - -- file: mode=01777 path="{{ swift_install_reop_dir }}/{{swift_install_repo_name}}" diff --git a/install/swift/templates/cluster_config b/install/swift/templates/cluster_config deleted file mode 100644 index da900301..00000000 --- a/install/swift/templates/cluster_config +++ /dev/null @@ -1,83 +0,0 @@ -{ - "groups" : { - "keystone": [ "{{ swift_ip }}" ], - "swift-proxy": [ "{{ swift_ip }}" ], - "swift-md": [ "{{ swift_ip }}" ], - "swift-object": [ "{{ swift_ip }}" ], - "swift-ring-builder": [ "{{ swift_ip }}" ] - }, - {{ swift_ip | to_json }} : { - "rings_info": { - "ip" : "{{ swift_ip }}", - "zone": "1", - "region": "1" - }, - "swift_devices": { - "object_devices": [ - { "name" : "{{ swift_device }}", "weight": "100", "fs":"xfs", "port": "6200" } - ], - "container_devices": [ - { "name" : "{{ swift_device }}", "weight": "100", "fs":"xfs", "port": "6201" } - ], - "account_devices" : [ - { "name" : "{{ swift_device }}", "weight": "100", "fs":"xfs", "port": "6202" } - ] - }, - "ansible_ssh_user" : "root", - "internal_ip" : "{{ swift_ip }}" - }, - "keystone" : { - "vars": { - "keystone_endpoint_host": "{{ swift_ip }}", - "keystone_endpoint_port": "35357", - "keystone_auth_protocol": "http", - "keystone_internal_url": "http://{{ swift_ip }}:5000/v2.0", - "keystone_admin_url": "http://{{ swift_ip }}:35357/v2.0", - "keystone_public_url": "http://{{ swift_ip }}:5000/v2.0", - "keystone_admin_project": "service", - "keystone_admin_token": "ADMIN", - "keystone_admin_user": "admin", - "keystone_admin_tenant": "admin", - "keystone_admin_password": "admin", - "swift_default_tenant": "test", - "swift_default_user_name": "tester", - "swift_default_user_password": "testing", - "openstack_region" : "1", - "swift_public_endpoint": "{{ swift_ip }}", - "swift_management_endpoint": "{{ swift_ip }}", - "swift_internal_endpoint": "{{ swift_ip }}", - "proxy_port": "80" - } - }, - "swift": { - "vars": { - "swift_hash_path_prefix": "d55ca1881f1e09b1", - "swift_hash_path_suffix": "a3f3c381c916a198", - "log_swift_statsd": true, - "openstack_version": "mitaka", - "installation_source": "git", - "swift_git": "https://github.com/openstack/swift.git", - "swift_git_dir": "/tmp/git/swift", - "swift_git_tag": "HEAD" - } - }, - "swift-ring-builder": { - "ring_builder" : { - "account": { - "min_part_hours": "1", - "part_power": "18", - "replicas": "1" - }, - "object": { - "min_part_hours": "1", - "part_power": "18", - "replicas": "1" - }, - "container": { - "min_part_hours": "1", - "part_power": "18", - "replicas": "1" - } - } - } -} diff --git a/install/swift/vars.yml-sample b/install/swift/vars.yml-sample deleted file mode 100644 index ca8d7f59..00000000 --- a/install/swift/vars.yml-sample +++ /dev/null @@ -1,22 +0,0 @@ -#--------------------------------------------------------------------------- -# Copyright IBM Corp. 2015, 2015 All Rights Reserved -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# Limitations under the License. -#--------------------------------------------------------------------------- -swift_install_reop_dir: -#swift_install_repo_url: https://github.com/Open-I-Beam/swift-install.git -swift_install_repo_url: https://github.com/eranr/storlets-swift-install.git -swift_install_repo_name: swift-install -#swift_install_repo_branch: storlet-swift-install -swift_install_repo_branch: master -swift_device: -swift_ip: diff --git a/s2aio.sh b/s2aio.sh index e19a1582..358a079c 100755 --- a/s2aio.sh +++ b/s2aio.sh @@ -37,9 +37,6 @@ if [ ! -f ~/.ssh/id_rsa.pub ]; then ssh-keygen -q -t rsa -f ~/.ssh/id_rsa -N "" fi -# TODO: move gcc to swift-installation -sudo apt-get install -y gcc --force-yes - if [ "$TARGET" == "docker" ]; then # install docker sudo apt-get install apt-transport-https aufs-tools=1:3.2+20130722-1.1 linux-image-extra-`uname -r` -y --force-yes @@ -61,6 +58,7 @@ if [ "$TARGET" == "docker" ]; then fi export S2AIO_IP=`sudo docker exec s2aio ifconfig | grep "inet addr" | head -1 | awk '{print $2}' | awk -F":" '{print $2}'` + sudo docker exec s2aio sh -c "echo deb http://us.archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse >> /etc/apt/sources.list" sudo docker exec s2aio apt-get update sudo docker exec s2aio apt-get install software-properties-common -y --force-yes sudo docker exec s2aio apt-add-repository -y ppa:ansible/ansible @@ -77,11 +75,18 @@ if [ "$TARGET" == "docker" ]; then ssh-keygen -R $S2AIO_IP -f ~/.ssh/known_hosts ssh-keyscan -H $S2AIO_IP >> ~/.ssh/known_hosts - # Install Swift - cd install/swift - ./install_swift.sh docker $S2AIO_IP loop0 + sudo docker exec s2aio useradd stack + sudo docker exec s2aio mkdir /home/stack + sudo docker exec s2aio bash -c 'grep -q "^#includedir.*/etc/sudoers.d" /etc/sudoers ||\ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers' + sudo docker exec s2aio bash -c '( umask 226 && echo "stack ALL=(ALL) NOPASSWD:ALL" >\ + /etc/sudoers.d/50_stack_sh )' + sudo docker cp install/swift/install_swift.sh s2aio:/home/stack/install_swift.sh + sudo docker cp install/swift/localrc.sample s2aio:/home/stack/localrc.sample + sudo docker exec s2aio chown -R stack:stack /home/stack + sudo docker exec --user stack s2aio chmod -R 0755 /home/stack + sudo docker exec --user stack s2aio /home/stack/install_swift.sh docker $S2AIO_IP $FLAVOR sudo docker exec s2aio service rsyslog restart - cd - else export S2AIO_IP='127.0.0.1' @@ -106,7 +111,7 @@ else # Install Swift cd install/swift - ./install_swift.sh host $S2AIO_IP loop0 + ./install_swift.sh host $S2AIO_IP $FLAVOR cd - fi