Fix apt fails due to linux image missing on AWS EC2

`uname -r` doesn't show the kernel number with 'genric' suffix (it will
be as 'aws' suffix) on AWS EC2 so that we may need a more general way to
get the correct package name.

Change-Id: Ieaa6891cb441124f13b5dc822f6511bc3091e188
Closes-Bug: #1686256

doc/source/engine_dev_installation.rst

@@ -31,7 +31,7 @@ Create a localrc file under the devstack repository root directory:
     SERVICE_PASSWORD=$ADMIN_PASSWORD
 
     OS_IDENTITY_API_VERSION=2
-    OS_AUTH_URL="http://$KEYSTONE_IP:5000/v3"
+    OS_AUTH_URL="http://$KEYSTONE_IP/identity/v3"
     OS_USERNAME=$ADMIN_USER
     OS_USER_DOMAIN_ID=default
     OS_PASSWORD=$ADMIN_PASSWORD
@@ -74,7 +74,7 @@ defining some environment variables:
 ::
 
     export OS_IDENTITY_API_VERSION=2
-    export OS_AUTH_URL="http://$KEYSTONE_IP:5000/v3"
+    export OS_AUTH_URL="http://$KEYSTONE_IP/identity/v3"
     export OS_USERNAME=$ADMIN_USER
     export OS_USER_DOMAIN_ID=default
     export OS_PASSWORD=$ADMIN_PASSWORD
@@ -98,7 +98,7 @@ A convenient way to do so is to edit the user's .bashrc adding the lines:
     export OS_USERNAME=tester
     export OS_PASSWORD=testing
     export OS_TENANT_NAME=test
-    export OS_AUTH_URL=http://127.0.0.1:5000/v3
+    export OS_AUTH_URL=http://127.0.0.1/identity/v3
 
 Now do:
 
@@ -280,7 +280,7 @@ First, we get the tenant id. Using the Swift cli and the above create user do:
 
 ::
 
-    swift --os-auth-url http://127.0.0.1:5000/v3 --os-project-name test --os-project-domain-name default --os-username tester --os-password testing stat
+    swift --os-auth-url http://127.0.0.1/identity/v3 --os-project-name test --os-project-domain-name default --os-username tester --os-password testing stat
 
 The response from the above contains the account line, e.g.:
 
@@ -437,7 +437,7 @@ We use the swift cli as follows:
 ::
 
   swift post \
-  --os-auth-url=http://127.0.0.1:5000/v3 \
+  --os-auth-url=http://127.0.0.1/identity/v3 \
   --os-username=tester \
   --os-password=testing \
   --os-project-name=test \
@@ -446,7 +446,7 @@ We use the swift cli as follows:
 
 
   swift post \
-  --os-auth-url=http://127.0.0.1:5000/v3 \
+  --os-auth-url=http://127.0.0.1/identity/v3 \
   --os-username=tester \
   --os-password=testing \
   --os-project-name=test \
@@ -454,7 +454,7 @@ We use the swift cli as follows:
   storlet
 
   swift post \
-  --os-auth-url=http://127.0.0.1:5000/v3 \
+  --os-auth-url=http://127.0.0.1/identity/v3 \
   --os-username=tester \
   --os-password=testing \
   --os-project-name=test \
@@ -462,7 +462,7 @@ We use the swift cli as follows:
   dependency
 
   swift post \
-  --os-auth-url=http://127.0.0.1:5000/v3 \
+  --os-auth-url=http://127.0.0.1/identity/v3 \
   --os-username=tester \
   --os-password=testing \
   --os-project-name=test \
@@ -488,7 +488,7 @@ content.
         "all" : {
             "docker_device": "/home/docker_device",
             "storlet_source_dir": "~/storlets",
-            "keystone_public_url": "http://127.0.0.1:5000/v3",
+            "keystone_public_url": "http://127.0.0.1/identity/v3",
             "swift_endpoint_host": "127.0.0.1",
             "swift_public_url": "http://127.0.0.1:8080/v1",
             "storlets_enabled_attribute_name": "storlet-enabled",

install/storlets/roles/create_default_project_docker_image/tasks/main.yml

@@ -15,7 +15,7 @@
 
 # TODO(eranr): Change the below to use Ansible os_project_facts
 - name: Get the project id from Keystone
-  shell: openstack --os-auth-url {{ keystone_public_url }} --os-username {{ keystone_admin_user }} --os-password {{ keystone_admin_password }} --os-project-name {{ keystone_admin_project }} --os-default-domain-name {{ keystone_default_domain }} project list | grep -w {{ project_name }} | awk '{print $2}'
+  shell: openstack --os-auth-url {{ keystone_public_url }} --os-identity-api-version {{ keystone_auth_version }} --os-username {{ keystone_admin_user }} --os-password {{ keystone_admin_password }} --os-project-name {{ keystone_admin_project }} --os-default-domain-name {{ keystone_default_domain }} project list | grep -w {{ project_name }} | awk '{print $2}'
   register: project_id
 
 - name: Print the project id whose container we are processing

install/storlets/roles/docker_client/tasks/main.yml

@@ -38,7 +38,7 @@
 
 - name: install necessary linux-image-extra
   apt:
-    name: linux-image-extra-{{ uname.stdout_lines[0] }}
+    name: linux-image-generic-lts-xenial
     update-cache: yes
     force: yes
   when: "'0' in linux_image_extra.stdout_lines[0]"

install/storlets/roles/enable_default_swift_account_for_storlets/tasks/main.yml

@@ -68,12 +68,12 @@
            --write-acl "{{ storlet_management_account }}:{{ storlet_management_admin_username }}"
            docker_images
 
-- name: test if need to create member user
+#- name: test if need to create member user
-  shell: openstack --os-auth-url {{ keystone_public_url }} --os-username {{ keystone_admin_user }} --os-password {{ keystone_admin_password }} --os-project-name {{ keystone_admin_project }} --os-default-domain-name {{ keystone_default_domain }} user list --project "{{ storlets_default_project_name }}" | grep "{{ storlets_default_project_member_user }}" | wc -l
+#  shell: openstack --os-auth-url {{ keystone_admin_url }} --os-identity-api-version {{ keystone_auth_version }} --os-username {{ keystone_admin_user }} --os-password {{ keystone_admin_password }} --os-project-name {{ keystone_admin_project }} --os-default-domain-name {{ keystone_default_domain }} user list --project {{ storlets_default_project_name }} | grep {{ storlets_default_project_member_user }} | wc -l
-  register: member_user
+#  register: member_user
 
-- debug: msg="{{ member_user }}"
+#- debug: msg={{ member_user }}
 
-- name: Create member user
+#- name: Create member user
-  shell: openstack --os-auth-url {{ keystone_public_url }} --os-username {{ keystone_admin_user }} --os-password {{ keystone_admin_password }} --os-project-name {{ keystone_admin_project }} --os-default-domain-name {{ keystone_default_domain }} user create --project "{{ storlets_default_project_name }}" --password "{{ storlets_default_project_member_password }}" --enable "{{ storlets_default_project_member_user }}"
+#  shell: openstack --os-auth-url {{ keystone_admin_url }} --os-identity-api-version {{ keystone_auth_version }} --os-username {{ keystone_admin_user }} --os-password {{ keystone_admin_password }} --os-project-name {{ keystone_admin_project }} --os-default-domain-name {{ keystone_default_domain }} user create --project {{ storlets_default_project_name }} --password {{ storlets_default_project_member_password }} --enable {{ storlets_default_project_member_user }}
-  when: "'0' in member_user.stdout"
+#  when: "'0' in member_user.stdout"

install/storlets/templates/cluster_config

@@ -13,10 +13,12 @@
         "storlet_gateway_conf_file": "/etc/swift/storlet_docker_gateway.conf",
         "keystone_endpoint_host": "{{ keystone_endpoint_host }}",
         "keystone_public_url": "http://{{ keystone_endpoint_host }}/identity/v3",
+        "keystone_admin_url": "http://{{ keystone_endpoint_host }}/identity_admin",
         "keystone_admin_password": "{{ keystone_admin_password }}",
         "keystone_admin_user": "{{keystone_admin_user}}",
         "keystone_admin_project": "{{keystone_admin_project}}",
         "keystone_default_domain": "{{keystone_default_domain}}",
+        "keystone_auth_version": "3",
         "swift_endpoint_host": "{{ swift_endpoint_host }}",
         "swift_run_time_user" : "{{ swift_run_time_user }}",
         "swift_run_time_group" : "{{ swift_run_time_group }}",

install/swift/install_swift.sh

@@ -53,6 +53,10 @@ stop_swift
 set -u
 cd -
 
+# Make sure all keystone services are up.
+/usr/local/bin/uwsgi /etc/keystone/keystone-uwsgi-public.ini &> /dev/null &
+/usr/local/bin/uwsgi /etc/keystone/keystone-uwsgi-admin.ini &> /dev/null &
+
 # add tester, testing, test which is admin
 source $DEVSTACK_DIR/localrc
 project_test_created=$(openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | wc -l)
@@ -64,6 +68,11 @@ if [ $user_tester_created -eq 0 ]; then
     openstack user create --project $SWIFT_DEFAULT_PROJECT --password $SWIFT_DEFAULT_USER_PWD $SWIFT_DEFAULT_USER
     openstack role add --user $SWIFT_DEFAULT_USER --project $SWIFT_DEFAULT_PROJECT admin
 fi
+user_member_created=$(openstack user list | grep -w $SWIFT_MEMBER_USER | wc -l)
+if [ $user_member_created -eq 0 ]; then
+    openstack user create --project $SWIFT_DEFAULT_PROJECT --password $SWIFT_MEMBER_USER_PWD $SWIFT_MEMBER_USER
+    openstack role add --user $SWIFT_MEMBER_USER --project $SWIFT_DEFAULT_PROJECT _member_
+fi
 
 # add entry to fstab
 mount_added=$(grep swift.img /etc/fstab | wc -l)

install/swift/localrc.sample

@@ -31,3 +31,5 @@ SWIFT_HASH=1234567890
 SWIFT_DEFAULT_PROJECT=test
 SWIFT_DEFAULT_USER=tester
 SWIFT_DEFAULT_USER_PWD=testing
+SWIFT_MEMBER_USER=tester_member
+SWIFT_MEMBER_USER_PWD=member

s2aio.sh

@@ -39,7 +39,7 @@ fi
 
 if [ "$TARGET" == "docker" ]; then
     # install docker
-    sudo apt-get install apt-transport-https aufs-tools linux-image-extra-`uname -r` -y --force-yes
+    sudo apt-get install apt-transport-https aufs-tools linux-image-generic-lts-xenial -y --force-yes
     sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
     sudo sh -c "echo deb https://apt.dockerproject.org/repo ubuntu-xenial main > /etc/apt/sources.list.d/docker.list"
     sudo apt-get update
@@ -129,5 +129,5 @@ sudo chown $USER:$USER cluster_config.json
 echo "export OS_IDENTITY_API_VERSION=3" >> ~/.bashrc
 echo "export OS_USERNAME=tester; export OS_PASSWORD=testing" >> ~/.bashrc
 echo "export OS_PROJECT_NAME=test; OS_DEFAULT_DOMAIN=default" >> ~/.bashrc
-echo "export OS_AUTH_URL=http://"$S2AIO_IP":5000/v3" >> ~/.bashrc
+echo "export OS_AUTH_URL=http://"$S2AIO_IP"/identity/v3" >> ~/.bashrc
 set +eu

tests/unit/tools/extensions/test_ipython.py

@@ -75,7 +75,7 @@ class BaseTestIpythonExtension(object):
     def _set_auth_environ(self):
         # helper method to set auth information for keystone v3 (default)
         os.environ['OS_AUTH_VERSION'] = '3'
-        os.environ['OS_AUTH_URL'] = 'http://127.0.0.1:5000/v3'
+        os.environ['OS_AUTH_URL'] = 'http://127.0.0.1/identity/v3'
         os.environ['OS_USERNAME'] = 'tester'
         os.environ['OS_PASSWORD'] = 'testing'
         os.environ['OS_USER_DOMAIN_NAME'] = 'default'