diff --git a/contrib/devstack-heat/README.rst b/contrib/devstack-heat/README.rst index 21a9580ec..831d6b0f7 100644 --- a/contrib/devstack-heat/README.rst +++ b/contrib/devstack-heat/README.rst @@ -1,10 +1,19 @@ -Kuryr Heat Templates +Devstack Heat Templates ==================== This set of scripts and Heat templates are useful for deploying devstack scenarios. It handles the creation of an allinone devstack nova instance and its networking needs. +It currently supports: + +* devstack-heat: For "baremetal" deployments. It will create a VM that acts as + if it were baremetal, i.e., with all the openstack services and with its own + overlay. +* devstack-overcloud-heat: For pod-in-VM deployments. It will create a VM and + deploy only k8s and kuryr-kubernetes, using the Neutron and Neutron-lbaasv2 + from your OpenStack cloud. + Prerequisites ~~~~~~~~~~~~~ @@ -13,8 +22,13 @@ Packages to install on the host you run devstack-heat (not on the cloud server): * jq * openstack-cli -If you want to run devstack from the master commit, this application requires a -github token due to the github api rate limiting: +If you intend to use devstack-overcloud-heat. Make sure that your OpenStack +cloud runs Neutron LBaaSv2 and either select **macvlan** (support will be +added soon) mode **OR** ensure it supports **Neutron trunk ports**. + +If you want to run devstack from the master commit, this application can +optionally use a github token to speed up the lookup of which is the most recent +master commit. You can generate one without any permissions at: @@ -25,12 +39,12 @@ so: echo "export DEVSTACK_HEAT_GH_TOKEN=my_token" >> ~/.bashrc -After creating the instance, devstack-heat will immediately start creating a -devstack `stack` user and using devstack to stack kuryr-kubernetes. When it is -finished, there'll be a file names `/opt/stack/ready`. +After creating the instance, devstack-(overcloud)-heat will immediately start +creating a devstack `stack` user and using devstack to stack kuryr-kubernetes. +When it is finished, there'll be a file names `/opt/stack/ready`. -How to run -~~~~~~~~~~ +How to run devstack-heat +~~~~~~~~~~~~~~~~~~~~~~~~ In order to run it, make sure that you have sourced your OpenStack cloud provider openrc file and tweaked `hot/parameters.yml` to your liking then launch @@ -53,9 +67,43 @@ This will create a stack named *gerrit_465657*. Further devstack-heat subcommands should be called with the whole name of the stack, i.e., *gerrit_465657*. +How to run devstack-overcloud-heat +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It works almost exactly like devstack-heat. The differences are that if you want +it to use your undercloud keystone that was not set up by devstack (for example +if you deployed manually or with packstack), you should pass this extra +flag to stacking:: + + ./devstack-overcloud-heat stack --override-keystone 472763 + +If you do not pass the keystone override flag, it will use all the devstack +assumptions about keystone. apuimedo highly recommends using the override. + +In order to unstack, with devstack-heat, one could optionally just use:: + + openstack unstack name_of_my_stack + +**With overcloud that will not work**. This is because OpenStack Heat does not +have support for trunk ports yet. Thus, in order for this to work, what we do is +put Heat stack hooks on stacking and unstacking that halt stacking and +unstacking so that we can perform trunk operations using Neutron. For all this, +and until Heat trunk support is in and we change to use it. In order to +unstack, always use devstack-overcloud-heat:: + + ./devstack-overcloud-heat unstack gerrit_472763 + +It is very important that before you unstack, you have deleted all the subports +that kuryr controller created for you from the overcloud, otherwise unstacking +will fail complaining that the trunk can't be deleted due to the presence of +subports. + Getting inside the deployment ----------------------------- +The following can be used interchangeably with ./devstack-heat and +./devstack-overcloud-heat + You can then ssh into the deployment in two ways:: ./devstack-heat show name_of_my_stack diff --git a/contrib/devstack-heat/devstack-overcloud-heat b/contrib/devstack-heat/devstack-overcloud-heat new file mode 100755 index 000000000..c39d67daf --- /dev/null +++ b/contrib/devstack-heat/devstack-overcloud-heat @@ -0,0 +1,159 @@ +#!/bin/bash + +CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# shellcheck disable=SC1091 +# shellcheck source=lib/devstack-heat +source "${CURR_DIR}/lib/devstack-heat" + +function sub_stack() { + local deployment + local default_sg + local project_id + local ost_address + local rc + local use_devstack_keystone + + if [[ "$1" == "--override-keystone" ]]; then + use_devstack_keystone="false" + shift + else + use_devstack_keystone="true" + fi + + deployment=${1:-master_} + deployment=$(_generate_deployment_name "${deployment}") + + # create stack + _confirm_or_exit "Deploying the stack ${deployment}" + echo "Starting..." + + default_sg=$(_get_default_sg_uuid) + project_id=$(_get_project_id) + ost_address=$(_get_keystone_host) + + openstack stack create \ + -e hot_overcloud/parameters.yml \ + --parameter "pod_sg=$default_sg" \ + --parameter "project_id=$project_id" \ + --parameter "ost_address=$ost_address" \ + --parameter "use_devstack_keystone=$use_devstack_keystone" \ + --parameter "keystone_v3_url=$OS_AUTH_URL" \ + --parameter "kuryr_username=$OS_USERNAME" \ + --parameter "kuryr_password=$OS_PASSWORD" \ + --parameter "kuryr_project_name=$OS_PROJECT_NAME" \ + --parameter "kuryr_domain_name=$OS_USER_DOMAIN_NAME" \ + --parameter "kuryr_project_domain_name=$OS_PROJECT_DOMAIN_NAME" \ + -t hot_overcloud/deployment.yml \ + "$deployment" + + rc=$? + + + if [[ ! "$rc" == "0" ]]; then + exit "$rc" + fi + + printf "Waiting for the hook" + while true; do + hooks=$(openstack stack hook poll "${deployment}" | grep -c nodes) + if [ "$hooks" -eq 1 ]; then + break + fi + sleep 5 + printf "." + done + printf "\n" + + _create_trunks "$deployment" + + echo "Continue with the stacking" + openstack stack hook clear --pre-create "$deployment" nodes + + sub_show "$deployment" +} + +function sub_unstack() { + local trunks + local deployment + local hooks + + deployment=${1:-_} + if [[ "${deployment}" == "_" ]]; then + (>&2 echo "You must put the whole stack name for unstacking") + exit 1 + fi + + trunks=( $(openstack stack output show "${deployment}" trunk_ports \ + -f json | \ + jq -r '.output_value' | \ + jq -r '.[]') ) + + _confirm_or_exit "Deleting the stack ${deployment}" + openstack stack delete "$deployment" + + printf "Waiting for the hook" + while true; do + hooks=$(openstack stack hook poll "$deployment" | grep -c trunk_ports) + if [ "$hooks" -eq 1 ]; then + break + fi + sleep 5 + printf "." + done + printf "\n" + + for port in "${trunks[@]}"; do + echo "Deleting trunk for parent port $port" + openstack network trunk delete "trunk-${port}" + done + + echo "Continue with the unstacking" + openstack stack hook clear --pre-delete "$deployment" trunk_ports +} + +function sub_show() { + local deployment + deployment=${1:-_} + if [[ "${deployment}" == "_" ]]; then + (>&2 echo "You must put the whole stack name for showing the stack") + exit 1 + fi + + _wait_for_after_in_progress "${1:-master_}" + echo "VM subnet: $(openstack stack output show "${deployment}" vm_subnet -f json | jq -r '.output_value')" + echo "Nodes FIPs: $(openstack stack output show "${deployment}" node_fips -f json | jq -r '.output_value' | jq -r '.[]?' | xargs echo)" + printf "\n" +} + + + +function sub_help() { + local myname + myname=$(basename "$0") + + printf "Usage: %s [options]\n" "$myname" + printf "Subcommands:\n" + printf " stack gerrit_change_number Create Heat stack\n" + printf " unstack my_stack_name Delete Heat stack\n" + printf " show my_stack_name Show important info about the deployed Heat stack\n" + printf " getkey my_stack_name Output the Heat stack instances privkey to stdout\n" + printf " ssh my_stack_name gets the key and sshs into the stack user of the Stack\n" +} + +command=$1 +case $command in + "" | "-h" | "--help") + sub_help + ;; + *) + shift + "sub_${command}" "$@" + if [ $? = 127 ]; then + echo "Error: '$command' is not a known $(basename "$0") command." >&2 + echo " Run \'$(basename "$0") --help\' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac + diff --git a/contrib/devstack-heat/hot_overcloud/deployment.yml b/contrib/devstack-heat/hot_overcloud/deployment.yml new file mode 100644 index 000000000..41012e139 --- /dev/null +++ b/contrib/devstack-heat/hot_overcloud/deployment.yml @@ -0,0 +1,183 @@ +heat_template_version: 2014-10-16 + +description: Simple template to deploy kuryr resources + +parameters: + image: + type: string + label: Image name or ID + description: Image to be used for the kuryr nodes + master_flavor: + type: string + label: Flavor + description: Flavor to be used for the masters + default: m1.small + flavor: + type: string + label: Flavor + description: Flavor to be used for the workers + default: m1.small + public_net: + type: string + description: public network for the instances + default: public + vm_net_cidr: + type: string + description: vm_net network address (CIDR notation) + vm_net_gateway: + type: string + description: vm_net network gateway address + pod_net_cidr: + type: string + description: pod_net network address (CIDR notation) + pod_net_gateway: + type: string + description: pod_net network gateway address + service_net_cidr: + type: string + description: service_net network address (CIDR notation) + service_net_gateway: + type: string + description: service_net network gateway address + service_router_port_ip: + type: string + description: IP for service router port + default: 172.30.255.254 + service_lbaas_ip: + type: string + description: IP for the kubernetes load balancer + default: 172.30.0.1 + node_num: + type: number + description: Number of VMs + default: 1 + pod_sg: + type: string + description: Security group to assign to Pods + project_id: + type: string + label: Project UUID + description: Project UUID + binding_driver: + type: string + description: Kubernetes integration binding driver + default: kuryr.lib.binding.drivers.vlan + vif_driver: + type: string + description: Kubernetes controller VIF driver + default: nested-vlan + ost_address: + type: string + label: OSt controller address + description: Address of the OpenStack controller + use_devstack_keystone: + type: string + description: Whether Devstack Keystone should be used + default: true + keystone_v3_url: + type: string + description: auth URI to Keystone + default: https://myip:myport/v3 + kuryr_username: + type: string + description: Keystone username for the Kuryr Controller to use + default: kuryr + kuryr_password: + type: string + description: Keystone password for the Kuryr Controller to use + default: mypass + kuryr_project_name: + type: string + description: Keystone project name for the Kuryr Controller to use + default: demo + kuryr_domain_name: + type: string + description: Keystone domain name (kuryr user) for the Kuryr Controller to use + default: default + kuryr_project_domain_name: + type: string + description: Keystone domain name (kuryr project) for the Kuryr Controller to use + default: default + + +resources: + network: + type: OS::Kuryr::PodInVMNetworking + properties: + public_net: { get_param: public_net } + vm_net_cidr: { get_param: vm_net_cidr } + vm_net_gateway: { get_param: vm_net_gateway } + pod_net_cidr: { get_param: pod_net_cidr } + pod_net_gateway: { get_param: pod_net_gateway } + service_net_cidr: { get_param: service_net_cidr } + service_net_gateway: { get_param: service_net_gateway } + service_router_port_ip: { get_param: service_router_port_ip } + service_lbaas_ip: { get_param: service_lbaas_ip } + + master_key: + type: OS::Nova::KeyPair + properties: + name: { get_param: 'OS::stack_name' } + save_private_key: true + + trunk_ports: + type: OS::Heat::ResourceGroup + properties: + count: { get_param: node_num } + resource_def: + type: trunk.yaml + properties: + network: { get_attr: [network, vm_net_id] } + subnet: { get_attr: [network, vm_subnet_id] } + + nodes: + type: OS::Heat::ResourceGroup + properties: + count: { get_param: node_num } + resource_def: + type: OS::Kuryr::DevstackNode + properties: + public_net: { get_param: public_net } + image: { get_param: image } + flavor: { get_param: flavor } + key: { get_resource: master_key } + private_key: { get_attr: [master_key, private_key] } + public_key: { get_attr: [master_key, public_key] } + trunk_ports: { get_attr: [trunk_ports, trunk_port_id] } + vm_net: { get_attr: [network, vm_net_id] } + vm_subnet: { get_attr: [network, vm_subnet_id] } + service_subnet: { get_attr: [network, service_subnet_id] } + service_cidr: { get_param: service_net_cidr } + k8s_lb_pool: { get_attr: [network, k8s_lb_pool_id] } + k8s_api_sg: { get_attr: [network, k8s_api_sg_id] } + pod_subnet: { get_attr: [network, pod_subnet_id] } + pod_sg: { get_param: pod_sg } + project_id: { get_param: project_id } + binding_driver: { get_param: binding_driver } + vif_driver: { get_param: vif_driver } + ost_address: { get_param: ost_address } + use_devstack_keystone: { get_param: use_devstack_keystone } + keystone_v3_url: { get_param: keystone_v3_url } + kuryr_username: { get_param: kuryr_username } + kuryr_password: { get_param: kuryr_password } + kuryr_project_name: { get_param: kuryr_project_name } + kuryr_domain_name: { get_param: kuryr_domain_name } + kuryr_project_domain_name: { get_param: kuryr_project_domain_name } + index: "%index%" + name: + str_replace: + template: "__stack__/vm-%index%" + params: + __stack__: { get_param: 'OS::stack_name' } + change_number: { get_param: 'OS::stack_name' } +outputs: + node_fips: + value: { get_attr: [nodes, node_fip] } + vm_subnet: + value: { get_attr: [network, vm_subnet_id] } + master_key_pub: + value: { get_attr: [master_key, public_key] } + master_key_priv: + value: { get_attr: [master_key, private_key] } + trunk_ports: + value: { get_attr: [trunk_ports, trunk_port_id] } diff --git a/contrib/devstack-heat/hot_overcloud/distro_deps.sh b/contrib/devstack-heat/hot_overcloud/distro_deps.sh new file mode 100644 index 000000000..b8bd8e46e --- /dev/null +++ b/contrib/devstack-heat/hot_overcloud/distro_deps.sh @@ -0,0 +1,16 @@ +distro=$(awk -F'=' '/^ID=/ {print $2}' /etc/os-release) +distro="${distro%\"}" +distro="${distro#\"}" + +if [[ "$distro" =~ centos|fedora ]]; then + yum install -y git python-devel + yum group install -y Development Tools + if [[ "$distro" == "centos" ]]; then + yum install -y epel-release + sed -i -e '/Defaults requiretty/{ s/.*/# Defaults requiretty/ }' /etc/sudoers + fi + yum install -y jq + +elif [[ "$distro" =~ ubuntu|debian ]]; then + apt-get install -y build-essential git python-dev jq +fi diff --git a/contrib/devstack-heat/hot_overcloud/networking_deployment.yaml b/contrib/devstack-heat/hot_overcloud/networking_deployment.yaml new file mode 100644 index 000000000..935d85e58 --- /dev/null +++ b/contrib/devstack-heat/hot_overcloud/networking_deployment.yaml @@ -0,0 +1,210 @@ +heat_template_version: 2014-10-16 + +description: Simple template to deploy kuryr resources + +parameters: + public_net: + type: string + label: public net ID + description: Public network for the node FIPs + vm_net_cidr: + type: string + description: vm_net network address (CIDR notation) + vm_net_gateway: + type: string + description: vm_net network gateway address + pod_net_cidr: + type: string + description: pod_net network address (CIDR notation) + pod_net_gateway: + type: string + description: pod_net network gateway address + service_net_cidr: + type: string + description: service_net network address (CIDR notation) + service_net_gateway: + type: string + description: service_net network gateway address + service_router_port_ip: + type: string + description: IP for service router port + default: 172.30.255.254 + service_lbaas_ip: + type: string + description: IP for the kubernetes loadbalancer + default: 172.30.0.1 + +resources: + vm_net: + type: OS::Neutron::Net + properties: + name: + str_replace: + template: "__stack__/vm-net" + params: + __stack__: { get_param: 'OS::stack_name' } + + vm_subnet: + type: OS::Neutron::Subnet + properties: + network_id: { get_resource: vm_net } + cidr: { get_param: vm_net_cidr } + gateway_ip: { get_param: vm_net_gateway } + name: + str_replace: + template: "__stack__/vm-subnet" + params: + __stack__: { get_param: 'OS::stack_name' } + + pod_net: + type: OS::Neutron::Net + properties: + name: + str_replace: + template: "__stack__/pod-net" + params: + __stack__: { get_param: 'OS::stack_name' } + + pod_subnet: + type: OS::Neutron::Subnet + properties: + network_id: { get_resource: pod_net } + cidr: { get_param: pod_net_cidr } + gateway_ip: { get_param: pod_net_gateway } + enable_dhcp: False + name: + str_replace: + template: "__stack__/pod-subnet" + params: + __stack__: { get_param: 'OS::stack_name' } + + service_net: + type: OS::Neutron::Net + properties: + name: + str_replace: + template: "__stack__/service-net" + params: + __stack__: { get_param: 'OS::stack_name' } + + service_subnet: + type: OS::Neutron::Subnet + properties: + network_id: { get_resource: service_net } + cidr: { get_param: service_net_cidr } + gateway_ip: { get_param: service_net_gateway } + enable_dhcp: False + name: + str_replace: + template: "__stack__/service-subnet" + params: + __stack__: { get_param: 'OS::stack_name' } + + k8s_lb: + type: OS::Neutron::LBaaS::LoadBalancer + properties: + name: + str_replace: + template: "__stack__/kubernetes-lb" + params: + __stack__: { get_param: 'OS::stack_name' } + vip_address: { get_param: service_lbaas_ip } + vip_subnet: { get_resource: service_subnet } + + k8s_lb_listener: + type: OS::Neutron::LBaaS::Listener + properties: + name: + str_replace: + template: "__stack__/kubernetes-listener-tcp-443" + params: + __stack__: { get_param: 'OS::stack_name' } + loadbalancer: { get_resource: k8s_lb } + protocol: HTTPS + protocol_port: 443 + + k8s_lb_pool: + type: OS::Neutron::LBaaS::Pool + properties: + name: + str_replace: + template: "__stack__/kubernetes-https-pool" + params: + __stack__: { get_param: 'OS::stack_name' } + protocol: HTTPS + lb_algorithm: ROUND_ROBIN + listener: { get_resource: k8s_lb_listener } + + kuryr_router: + type: OS::Neutron::Router + properties: + external_gateway_info: + network: { get_param: public_net } + name: + str_replace: + template: "__stack__/router" + params: + __stack__: { get_param: 'OS::stack_name' } + + kr_vm_iface: + type: OS::Neutron::RouterInterface + properties: + router_id: { get_resource: kuryr_router } + subnet_id: { get_resource: vm_subnet } + + kr_pod_iface: + type: OS::Neutron::RouterInterface + properties: + router_id: { get_resource: kuryr_router } + subnet_id: { get_resource: pod_subnet } + + service_router_port: + type: OS::Neutron::Port + properties: + network: { get_resource: service_net} + fixed_ips: + - subnet: { get_resource: service_subnet } + ip_address: { get_param: service_router_port_ip } + name: service-router-port + + kr_service_iface: + type: OS::Neutron::RouterInterface + properties: + router_id: { get_resource: kuryr_router } + port: { get_resource: service_router_port } + + k8s_api_sg: + type: OS::Neutron::SecurityGroup + properties: + name: + str_replace: + template: "__stack__/k8s-api-sg" + params: + __stack__: { get_param: 'OS::stack_name' } + description: Ping and SSH + rules: + - protocol: icmp + - ethertype: IPv4 + remote_mode: remote_group_id + - ethertype: IPv6 + remote_mode: remote_group_id + - protocol: tcp + port_range_min: 22 + port_range_max: 22 + - protocol: tcp + port_range_min: 8443 + port_range_max: 8443 + +outputs: + vm_net_id: + value: { get_resource: vm_net } + vm_subnet_id: + value: { get_resource: vm_subnet } + pod_subnet_id: + value: { get_resource: pod_subnet } + service_subnet_id: + value: { get_resource: service_subnet } + k8s_lb_pool_id: + value: { get_resource: k8s_lb_pool } + k8s_api_sg_id: + value: { get_resource: k8s_api_sg } diff --git a/contrib/devstack-heat/hot_overcloud/node.yaml b/contrib/devstack-heat/hot_overcloud/node.yaml new file mode 100644 index 000000000..54d975268 --- /dev/null +++ b/contrib/devstack-heat/hot_overcloud/node.yaml @@ -0,0 +1,291 @@ +heat_template_version: 2014-10-16 + +description: Template to deploy openshift worker nodes + +parameters: + public_net: + type: string + label: public net ID + description: Public network for the node FIPs + image: + type: string + label: Image name or ID + description: Image to be used for the kuryr nodes + flavor: + type: string + label: Flavor + description: Flavor to be used for the image + default: m1.small + key: + type: string + label: key name + description: Keypair to be used for the instance + public_key: + type: string + label: key content for stack user authorized_keys + description: private key to configure all nodes + private_key: + type: string + label: key content to access other nodes + description: private key to configure all nodes + vm_net: + type: string + label: VM Network + description: Neutron network for VMs + vm_subnet: + type: string + label: VM Subnet + description: Neutron subnet for VMs + pod_subnet: + type: string + label: Pod Subnet + description: Neutron subnet for pods + service_subnet: + type: string + label: Service subnet + description: service subnet UUID + service_cidr: + type: string + label: Service CIDR + description: service_net network address (CIDR notation) + k8s_lb_pool: + type: string + label: K8s Pool + description: K8s LoadBalancer Pool + k8s_api_sg: + type: string + label: kubernetes API sg + description: Security Group for Kubernetes API + pod_sg: + type: string + label: Pod sg + description: Security Group for the pods + project_id: + type: string + label: Project UUID + description: Project UUID + name: + type: string + label: Instance name + description: devstack node instance name + change_number: + type: string + label: Gerrit change number + description: Either the gerrit change number or master-sha_of_the_commit + ost_address: + type: string + label: OSt controller address + description: Address of the OpenStack controller + trunk_ports: + type: comma_delimited_list + label: Trunk Ports + description: List of Neutron Trunk Ports for VMs + index: + type: number + binding_driver: + type: string + description: Kubernetes integration binding driver + default: kuryr.lib.binding.drivers.vlan + vif_driver: + type: string + description: Kubernetes controller VIF driver + default: nested-vlan + use_devstack_keystone: + type: string + description: Whether Devstack Keystone should be used + keystone_v3_url: + type: string + description: auth URI to Keystone + kuryr_username: + type: string + description: Keystone username for the Kuryr Controller to use + kuryr_password: + type: string + description: Keystone password for the Kuryr Controller to use + kuryr_project_name: + type: string + description: Keystone project name for the Kuryr Controller to use + kuryr_domain_name: + type: string + description: Keystone domain name (kuryr user) for the Kuryr Controller to use + kuryr_project_domain_name: + type: string + description: Keystone domain name (kuryr project) for the Kuryr Controller to use + + +resources: + instance_fip: + type: OS::Neutron::FloatingIP + properties: + floating_network: { get_param: public_net } + port_id: { get_param: [trunk_ports, {get_param: index}] } + + instance: + type: OS::Nova::Server + properties: + name: { get_param: name } + image: { get_param: image } + flavor: { get_param: flavor } + key_name: { get_param: key } + networks: + - port: { get_param: [trunk_ports, {get_param: index}]} + user_data_format: RAW + user_data: + str_replace: + params: + __distro_deps__: { get_file: distro_deps.sh } + __change_number__: { get_param: change_number } + __pubkey__: { get_param: public_key } + __undercloud_default_sg__: { get_param: pod_sg } + __pod_subnet_uuid__: { get_param: pod_subnet } + __undercloud_project_uuid__: { get_param: project_id } + __undercloud_service_subnet_uuid__: { get_param: service_subnet } + __undercloud_service_subnet_cidr__: { get_param: service_cidr } + __node_subnet_uuid__: { get_param: vm_subnet } + __vif_driver__: { get_param: vif_driver } + __binding_driver__: { get_param: binding_driver } + __ost_address__: { get_param: ost_address } + __kuryr_use_devstack_keystone__: { get_param: use_devstack_keystone } + __keystone_v3_endpoint__: { get_param: keystone_v3_url } + __keystone_kuryr_username__: { get_param: kuryr_username } + __keystone_kuryr_password__: { get_param: kuryr_password } + __keystone_kuryr_domain_name__: { get_param: kuryr_domain_name } + __keystone_kuryr_project_name__: { get_param: kuryr_project_name } + __keystone_kuryr_project_domain_name__: { get_param: kuryr_project_domain_name } + template: | + #!/bin/bash + set -ex + + # Wait a bit for connectivity + sleep 10 + + # Deps for devstack + __distro_deps__ + + # Stack user config + useradd -s /bin/bash -d /opt/stack -m stack + mkdir /opt/stack/.ssh + cat > /opt/stack/.ssh/authorized_keys << EOF + __pubkey__ + EOF + echo "stack ALL=(ALL) NOPASSWD: ALL" | tee /etc/sudoers.d/stack + + # Stacking + sudo -i -u stack /bin/bash - <<"EOF" + function get_from_gerrit() { + local stackname + local change_number + local ref + + stackname="__change_number__" + change_number=${stackname#*_} + echo "Finding latest ref for change ${change_number}" + ref=$(curl -s "https://review.openstack.org/changes/${change_number}?o=CURRENT_REVISION" | tail -n +2 | jq -r '.revisions[].ref') + echo "Fetching ref ${ref}" + git fetch https://git.openstack.org/openstack/kuryr-kubernetes "${ref}" && git checkout FETCH_HEAD + } + + function get_from_sha() { + local commit_sha + + commit_sha=$(echo "__change_number__" | cut -d'_' -f2) + echo "Sha to fetch: ${commit_sha}" + git checkout "$commit_sha" + } + + cd /opt/stack + git clone https://git.openstack.org/openstack-dev/devstack + git clone https://github.com/openstack/kuryr-kubernetes + pushd kuryr-kubernetes + + if [[ "__change_number__" =~ ^master.* ]]; then + get_from_sha + elif [[ "__change_number__" =~ ^gerrit.* ]]; then + get_from_gerrit + else + echo "Unrecognized stack name. Quitting..." + exit 1 + fi + popd + pushd devstack + + # The change is already downloaded, do not reclone + cat > /tmp/localconf.replacements << SEDEOF + s/# RECLONE=/RECLONE=/ + SEDEOF + + if [[ ! "__kuryr_use_devstack_keystone__" == "true" ]]; then + # We override Keystone AUTH to talk to Neutron + cat >> /tmp/localconf.replacements << SEDEOF + s/UNDERCLOUD_CONTROLLER_IP/__ost_address__/ + s/#KURYR_USE_UNDERCLOUD_DEVSTACK_KEYSTONE_CONFIG=/KURYR_USE_UNDERCLOUD_DEVSTACK_KEYSTONE_CONFIG=/ + s%#KURYR_UNDERCLOUD_KEYSTONE_URI=.*%KURYR_UNDERCLOUD_KEYSTONE_URI=__keystone_v3_endpoint__% + s/#KURYR_UNDERCLOUD_KEYSTONE_KURYR_USERNAME=.*/KURYR_UNDERCLOUD_KEYSTONE_KURYR_USERNAME=__keystone_kuryr_username__/ + s/#KURYR_UNDERCLOUD_KEYSTONE_KURYR_PASSWORD=.*/KURYR_UNDERCLOUD_KEYSTONE_KURYR_PASSWORD=__keystone_kuryr_password__/ + s/#KURYR_UNDERCLOUD_KEYSTONE_KURYR_DOMAIN_NAME=.*/KURYR_UNDERCLOUD_KEYSTONE_KURYR_DOMAIN_NAME=__keystone_kuryr_domain_name__/ + s/#KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_NAME=.*/KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_NAME=__keystone_kuryr_project_name__/ + s/#KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_DOMAIN_NAME=.*/KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_DOMAIN_NAME=__keystone_kuryr_project_domain_name__/ + s/#KURYR_UNDERCLOUD_KURYR_SERVICE_SUBNET=.*/KURYR_UNDERCLOUD_KURYR_SERVICE_SUBNET=__undercloud_service_subnet_uuid__/ + SEDEOF + fi + sed -f /tmp/localconf.replacements \ + ~/kuryr-kubernetes/devstack/local.conf.pod-in-vm.overcloud.sample \ + > /opt/stack/devstack/local.conf + + cat >> /opt/stack/devstack/local.conf << EOLOCALCONF + KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE=False + KURYR_CONFIGURE_NEUTRON_DEFAULTS=False + KURYR_K8S_CLUSTER_IP_RANGE=__undercloud_service_subnet_cidr__ + + [[post-config|/etc/kuryr/kuryr.conf]] + [neutron_defaults] + ovs_bridge=br-int + pod_security_groups=__undercloud_default_sg__ + pod_subnet=__pod_subnet_uuid__ + project=__undercloud_project_uuid__ + service_subnet=__undercloud_service_subnet_uuid__ + + [pod_vif_nested] + worker_nodes_subnet=__node_subnet_uuid__ + + [kubernetes] + pod_vif_driver=__vif_driver__ + + [binding] + driver=__binding_driver__ + link_iface = eth0 + EOLOCALCONF + + ./stack.sh + popd + + # Put keystonerc for the VM user to be able to check neutron + cat > /opt/stack/keystonerc << KEYSTONEEOF + unset OS_SERVICE_TOKEN + export OS_USERNAME=__keystone_kuryr_username__ + export OS_PASSWORD=__keystone_kuryr_password__ + export OS_AUTH_URL=__keystone_v3_endpoint__ + + export OS_PROJECT_NAME=__keystone_kuryr_project_name__ + export OS_USER_DOMAIN_NAME=__keystone_kuryr_domain_name__ + export OS_PROJECT_DOMAIN_NAME=__keystone_kuryr_project_domain_name__ + export OS_IDENTITY_API_VERSION=3 + KEYSTONEEOF + + touch ready + EOF + + + lb_member: + type: OS::Neutron::LBaaS::PoolMember + properties: + pool: { get_param: k8s_lb_pool } + protocol_port: 6443 + address: { get_attr: [instance, first_address]} + subnet: { get_param: vm_subnet } + + +outputs: + node_fip: + description: FIP address of the node + value: { get_attr: [instance_fip, floating_ip_address] } diff --git a/contrib/devstack-heat/hot_overcloud/parameters.yml b/contrib/devstack-heat/hot_overcloud/parameters.yml new file mode 100644 index 000000000..65a7cd704 --- /dev/null +++ b/contrib/devstack-heat/hot_overcloud/parameters.yml @@ -0,0 +1,23 @@ +parameters: + image: centos7 + flavor: m1.medium + public_net: public + vm_net_cidr: 10.11.0.0/24 + vm_net_gateway: 10.11.0.1 + pod_net_cidr: 10.10.0.0/16 + pod_net_gateway: 10.10.0.1 + service_net_cidr: 172.30.0.0/16 + service_net_gateway: 172.30.255.254 + service_router_port_ip: 172.30.255.254 + service_lbaas_ip: 172.30.0.1 + binding_driver: kuryr.lib.binding.drivers.vlan + vif_driver: nested-vlan + +resource_registry: + OS::Kuryr::PodInVMNetworking: networking_deployment.yaml + OS::Kuryr::DevstackNode: node.yaml + resources: + nodes: + hooks: pre-create + trunk_ports: + hooks: pre-delete diff --git a/contrib/devstack-heat/hot_overcloud/trunk.yaml b/contrib/devstack-heat/hot_overcloud/trunk.yaml new file mode 100644 index 000000000..d16b99c83 --- /dev/null +++ b/contrib/devstack-heat/hot_overcloud/trunk.yaml @@ -0,0 +1,31 @@ +heat_template_version: 2014-10-16 + +description: Template to create ports to be parent trunk ports + +parameters: + network: + type: string + label: VM Network + description: Neutron network for VMs + subnet: + type: string + label: VM Subnet + description: Neutron subnet for VMs + +resources: + trunk_port: + type: OS::Neutron::Port + properties: + network: { get_param: network } + security_groups: + - default + fixed_ips: + - subnet: { get_param: subnet } + +outputs: + trunk_port_id: + value: { get_resource: trunk_port } + trunk_port_ips: + value: { get_attr: [trunk_port, fixed_ips] } + trunk_port_dns: + value: { get_attr: [trunk_port, dns_assignment] } diff --git a/contrib/devstack-heat/lib/devstack-heat b/contrib/devstack-heat/lib/devstack-heat index 6dcc2e09e..341b6cf0f 100644 --- a/contrib/devstack-heat/lib/devstack-heat +++ b/contrib/devstack-heat/lib/devstack-heat @@ -30,8 +30,8 @@ function sub_ssh() { local key local fip - deployment=${1:-master_} - if [[ "${deployment}" == "master_" ]]; then + deployment=${1:-_} + if [[ "${deployment}" == "_" ]]; then (>&2 echo "You must put the whole stack name for logging into the node") exit 1 fi @@ -56,6 +56,7 @@ function _generate_deployment_name() { (>&2 echo "Didn't find a Github token in ENV var DEVSTACK_HEAT_GH_TOKEN. Falling back to cloning repo...") tmpdir=$(mktemp -d) git clone https://git.openstack.org/openstack/kuryr-kubernetes "${tmpdir}/kuryr-kubernetes" > /dev/null + pushd "${tmpdir}/kuryr-kubernetes" > /dev/null latest_commit=$(git rev-parse HEAD) popd > /dev/null @@ -86,3 +87,71 @@ function _confirm_or_exit() { exit 1 fi } + +function _get_sgs() { + local sg_debug_data + + sg_debug_data=${1:-get} + + if [[ "$sg_debug_data" == "get" ]]; then + sg_debug_data=$(openstack -vvv security group list 2>&1) + fi + + echo "$sg_debug_data" | \ + awk '/RESP BODY/ && /security_groups/ {$1=""; $2=""; print}' | \ + jq -r '.security_groups[] | "\(.id) \(.name) \(.project_id)"' +} + +function _get_default_sg_uuid() { + local sg_debug_data + local project_id + + sg_debug_data=${1:-get} + + if [[ "$sg_debug_data" == "get" ]]; then + sg_debug_data=$(openstack -vvv security group list 2>&1) + fi + + project_id=$(echo "$sg_debug_data" | \ + awk '$1 ~ /^{\"token/ {print $0; exit}' | \ + jq -r '.token.project.id') + + _get_sgs "$sg_debug_data" | \ + awk "/default/ && /$project_id/ { print \$1 }" +} + +function _get_keystone_host() { + echo "$OS_AUTH_URL" | \ + awk -F'/' '{print $3}' | \ + cut -f1 -d: +} + +function _get_project_id() { + local sg_debug_data + local project_id + + sg_debug_data=${1:-get} + + if [[ "$sg_debug_data" == "get" ]]; then + sg_debug_data=$(openstack -vvv security group list 2>&1) + fi + + echo "$sg_debug_data" | \ + awk '$1 ~ /^{\"token/ {print $0; exit}' | \ + jq -r '.token.project.id' +} + +function _create_trunks() { + local deployment + deployment=${1:-_} + + if [[ "${deployment}" == "_" ]]; then + (>&2 echo "You must put the whole stack name for getting the key") + exit 1 + fi + + (>&2 echo "creating trunks") + openstack port list -f value | \ + awk -v "port_name=${deployment}-trunk_ports" \ + '$0 ~ port_name {system("openstack network trunk create --parent-port " $1 " trunk-" $1)}' +} diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index 8849e6c40..d14fd833d 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -10,7 +10,6 @@ # Dependencies: # (none) - function ovs_bind_for_kubelet() { local port_id local port_mac @@ -245,3 +244,57 @@ function create_k8s_subnet { || die $LINENO \ "Failed to enable routing for K8s ${subnet_name} subnet" } + +# _multihost_override_devstack_keystone +# Description: True iif we are in multi host stacking and we should not use +# devstack's keystone assumptions. +function _multihost_override_devstack_keystone() { + local multi_host + local use_devstack_keystone + local rc + + multi_host=$(trueorfalse False MULTI_HOST) + use_devstack_keystone=$(trueorfalse True KURYR_USE_UNDERCLOUD_DEVSTACK_KEYSTONE_CONFIG) + if [[ ( "$multi_host" == "True" ) && \ + ( "$use_devstack_keystone" == "False" ) ]]; then + rc=0 + else + rc=1 + fi + + return $rc +} + +# _openstack_client +# Description: Uses your undercloud config for openstack client if you +# are overriding devstack's keystone assumptions. +# Params: +# args - OpenStack client parameters +function _openstack_client() { + if _multihost_override_devstack_keystone; then + openstack --os-auth-type password \ + --os-auth-url "${KURYR_UNDERCLOUD_KEYSTONE_URI}" \ + --os-username "${KURYR_UNDERCLOUD_KEYSTONE_KURYR_USERNAME}" \ + --os-password "${KURYR_UNDERCLOUD_KEYSTONE_KURYR_PASSWORD}" \ + --os-project-name "${KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_NAME}" \ + --os-domain-name "${KURYR_UNDERCLOUD_KEYSTONE_KURYR_DOMAIN_NAME}" \ + --os-project-domain-name "${KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_DOMAIN_NAME}" \ + "$@" + else + openstack --os-cloud devstack-admin \ + --os-region "$REGION_NAME" \ + "$@" + fi +} + +# _get_service_subnet +# Description: Returns the subnet id or name for services depending on +# whether the deployment is multi host with keystone override or +# not. +function _get_service_subnet() { + if _multihost_override_devstack_keystone; then + echo "$KURYR_UNDERCLOUD_KURYR_SERVICE_SUBNET" + else + echo "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" + fi +} diff --git a/devstack/local.conf.pod-in-vm.overcloud.sample b/devstack/local.conf.pod-in-vm.overcloud.sample index 7aea8ceb2..0030cecfc 100644 --- a/devstack/local.conf.pod-in-vm.overcloud.sample +++ b/devstack/local.conf.pod-in-vm.overcloud.sample @@ -23,6 +23,18 @@ MYSQL_HOST=$SERVICE_HOST RABBIT_HOST=$SERVICE_HOST KURYR_CONFIGURE_NEUTRON_DEFAULTS=True +# Optional section to override Keystone and Auth config +#KURYR_USE_UNDERCLOUD_DEVSTACK_KEYSTONE_CONFIG=False +#KURYR_UNDERCLOUD_KEYSTONE_URI=https://my_keystone_host:my_keystone_port/my_v3_path +#KURYR_UNDERCLOUD_KEYSTONE_KURYR_USERNAME=kuryr_username +#KURYR_UNDERCLOUD_KEYSTONE_KURYR_PASSWORD=kuryr_password +#KURYR_UNDERCLOUD_KEYSTONE_KURYR_DOMAIN_NAME=default +#KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_NAME=kuryr_project_name +#KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_DOMAIN_NAME=default +#KURYR_UNDERCLOUD_KURYR_SERVICE_SUBNET=k8s-service-subnet + + +KURYR_CONFIGURE_NEUTRON_DEFAULTS=False KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE=False # Include subnet pool id to use. You can use the one from the default undercloud devstack diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 428bf3422..373dd9de9 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -64,8 +64,18 @@ function configure_kuryr { # Neutron API server & Neutron plugin if is_service_enabled kuryr-kubernetes; then - configure_auth_token_middleware "$KURYR_CONFIG" kuryr \ - "$KURYR_AUTH_CACHE_DIR" neutron + if _multihost_override_devstack_keystone; then + iniset "$KURYR_CONFIG" neutron auth_type password + iniset "$KURYR_CONFIG" neutron auth_url "$KURYR_UNDERCLOUD_KEYSTONE_URI" + iniset "$KURYR_CONFIG" neutron username "$KURYR_UNDERCLOUD_KEYSTONE_KURYR_USERNAME" + iniset "$KURYR_CONFIG" neutron password "$KURYR_UNDERCLOUD_KEYSTONE_KURYR_PASSWORD" + iniset "$KURYR_CONFIG" neutron user_domain_name "$KURYR_UNDERCLOUD_KEYSTONE_KURYR_DOMAIN_NAME" + iniset "$KURYR_CONFIG" neutron project_name "$KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_NAME" + iniset "$KURYR_CONFIG" neutron project_domain_name "$KURYR_UNDERCLOUD_KEYSTONE_KURYR_PROJECT_DOMAIN_NAME" + else + configure_auth_token_middleware "$KURYR_CONFIG" kuryr \ + "$KURYR_AUTH_CACHE_DIR" neutron + fi fi } @@ -103,9 +113,7 @@ function create_k8s_router_fake_service { local fake_svc_name fake_svc_name='kuryr-svc-router' - service_cidr=$(openstack --os-cloud devstack-admin \ - --os-region "$REGION_NAME" \ - subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \ + service_cidr=$(_openstack_client subnet show "$(_get_service_subnet)" \ -c cidr -f value) router_ip=$(_cidr_range "$service_cidr" | cut -f2) existing_svc_ip=$(/usr/local/bin/kubectl get svc --namespace kube-system -o jsonpath='{.items[?(@.metadata.name=='"\"${fake_svc_name}\""')].spec.clusterIP}') @@ -240,9 +248,7 @@ function prepare_kubernetes_files { local service_cidr local k8s_api_clusterip - service_cidr=$(openstack --os-cloud devstack-admin \ - --os-region "$REGION_NAME" \ - subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\ + service_cidr=$(_openstack_client subnet show "$(_get_service_subnet)"\ -c cidr -f value) k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1) mountpoint=$(get_hyperkube_container_cacert_setup_dir "$KURYR_HYPERKUBE_VERSION") @@ -430,6 +436,7 @@ function run_kuryr_kubernetes { } +# shellcheck source=devstack/lib/kuryr_kubernetes source $DEST/kuryr-kubernetes/devstack/lib/kuryr_kubernetes # main loop @@ -441,7 +448,9 @@ if is_service_enabled kuryr-kubernetes; then fi elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then - create_kuryr_account + if ! _multihost_override_devstack_keystone; then + create_kuryr_account + fi configure_kuryr fi @@ -502,7 +511,9 @@ if is_service_enabled kuryr-kubernetes; then elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then create_k8s_router_fake_service - create_k8s_api_service + if ! _multihost_override_devstack_keystone; then + create_k8s_api_service + fi fi if [[ "$1" == "unstack" ]]; then diff --git a/devstack/settings b/devstack/settings index b87a97297..5988b5b33 100644 --- a/devstack/settings +++ b/devstack/settings @@ -43,3 +43,6 @@ KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-} # Kuryr_ovs_baremetal KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE=${KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE:-True} + +# Whether or not to assume Keystone has been deployed by devstack on the undercloud +KURYR_USE_UNDERCLOUD_DEVSTACK_KEYSTONE_CONFIG=${KURYR_USE_UNDERCLOUD_DEVSTACK_KEYSTONE_CONFIG:-True}