Docker compute role configured via Puppet

This change adds a containerized version of the overcloud compute node for
TripleO. Configuration files are generated via OpenStack Puppet modules
which are then used to externally configure kolla containers for
each OpenStack service.

See the README-containers.md file for more information on how to set this up.

This uses AtomicOS as a base operating system and requires that we bootstrap
the image with a container which contains the required os-collect-config agent
hooks to support running puppet, shell scripts, and docker compose.

Change-Id: Ic8331f52b20a041803a9d74cdf0eb81266d4e03c
Co-Authored-By: Ian Main <imain@redhat.com>
Co-Authored-By: Ryan Hallisey <rhallise@redhat.com>

docker/README-containers.md

@@ -0,0 +1,50 @@
+# Using Docker Containers With TripleO
+
+## Configuring TripleO with to use a container based compute node.
+
+Steps include:
+- Adding a base OS image to glance
+- Deploy an overcloud configured to use the docker compute heat templates
+
+## Getting base OS image working.
+
+Download the fedora atomic image into glance:
+
+```
+wget https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Atomic-22-20150521.x86_64.qcow2
+glance image-create --name fedora-atomic --file Fedora-Cloud-Atomic-22-20150521.x86_64.qcow2 --disk-format qcow2 --container-format bare
+```
+
+## Configuring TripleO
+
+You can use the tripleo.sh script up until the point of running the Overcloud.
+https://github.com/openstack/tripleo-common/blob/master/scripts/tripleo.sh
+
+Create the Overcloud:
+```
+$ openstack overcloud deploy --templates=tripleo-heat-templates -e tripleo-heat-templates/environments/docker-rdo.yaml --libvirt-type=qemu
+```
+
+Source the overcloudrc and then you can use the overcloud.
+
+## Debugging
+
+You can ssh into the controller/compute nodes by using the heat key, eg:
+```
+nova list
+ssh heat-admin@<compute_node_ip>
+```
+
+You can check to see what docker containers are running:
+```
+sudo docker ps -a
+```
+
+To enter a container that doesn't seem to be working right:
+```
+sudo docker exec -ti <container name> /bin/bash
+```
+
+Then you can check logs etc.
+
+You can also just do a 'docker logs' on a given container.

docker/compute-post.yaml

@@ -0,0 +1,228 @@
+heat_template_version: 2015-04-30
+
+description: >
+  OpenStack compute node post deployment for Docker.
+
+parameters:
+  servers:
+    type: json
+  NodeConfigIdentifiers:
+     type: json
+     description: Value which changes if the node configuration may need to be re-applied
+  DockerComputeImage:
+    type: string
+  DockerComputeDataImage:
+    type: string
+  DockerLibvirtImage:
+    type: string
+  DockerNeutronAgentImage:
+    type: string
+  DockerOpenvswitchImage:
+    type: string
+  DockerOvsVswitchdImage:
+    type: string
+  DockerOpenvswitchDBImage:
+    type: string
+
+resources:
+
+  ComputePuppetConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: puppet
+      options:
+        enable_hiera: True
+        enable_facter: False
+        tags: package,file,concat,file_line,nova_config,neutron_config,neutron_agent_ovs,neutron_plugin_ml2
+      inputs:
+      - name: tripleo::packages::enable_install
+        type: Boolean
+        default: True
+      outputs:
+      - name: result
+      config:
+        get_file: ../puppet/manifests/overcloud_compute.pp
+
+  ComputePuppetDeployment:
+    type: OS::Heat::SoftwareDeployments
+    properties:
+      servers:  {get_param: servers}
+      config: {get_resource: ComputePuppetConfig}
+      input_values:
+        update_identifier: {get_param: NodeConfigIdentifiers}
+        tripleo::packages::enable_install: True
+
+  CopyEtcConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      outputs:
+      - name: result
+      config: {get_file: ./copy-etc.sh}
+
+  CopyEtcDeployment:
+    type: OS::Heat::SoftwareDeployments
+    depends_on: ComputePuppetDeployment
+    properties:
+      config: {get_resource: CopyEtcConfig}
+      servers:  {get_param: servers}
+
+  NovaComputeContainersDeploymentOVS:
+    type: OS::Heat::StructuredDeployments
+    properties:
+      config: {get_resource: NovaComputeContainersConfigOVS}
+      servers: {get_param: servers}
+
+  NovaComputeContainersConfigOVS:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: docker-compose
+      config:
+        ovsvswitchd:
+          image: {get_param: DockerOvsVswitchdImage}
+          container_name: ovs-vswitchd
+          net: host
+          privileged: true
+          restart: always
+          volumes:
+           - /run:/run
+           - /lib/modules:/lib/modules:ro
+          environment:
+           - KOLLA_CONFIG_STRATEGY=CONFIG_EXTERNAL_COPY_ALWAYS
+
+        openvswitchdb:
+          image: {get_param: DockerOpenvswitchDBImage}
+          container_name: ovs-db-server
+          net: host
+          restart: always
+          volumes:
+           - /run:/run
+          environment:
+           - KOLLA_CONFIG_STRATEGY=CONFIG_EXTERNAL_COPY_ALWAYS
+
+  NovaComputeContainersDeploymentNetconfig:
+    type: OS::Heat::SoftwareDeployments
+    depends_on: NovaComputeContainersDeploymentOVS
+    properties:
+      config: {get_resource: NovaComputeContainersConfigNetconfig}
+      servers: {get_param: servers}
+
+  # We run os-net-config here because we depend on the ovs containers to be up
+  # and running before we configure the network.  This allows explicit timing
+  # of the network configuration.
+  NovaComputeContainersConfigNetconfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      outputs:
+      - name: result
+      config: |
+        #!/bin/bash
+        /usr/local/bin/run-os-net-config
+
+  LibvirtContainersDeployment:
+    type: OS::Heat::StructuredDeployments
+    depends_on: [CopyEtcDeployment, ComputePuppetDeployment, NovaComputeContainersDeploymentNetconfig]
+    properties:
+      config: {get_resource: LibvirtContainersConfig}
+      servers: {get_param: servers}
+
+  LibvirtContainersConfig:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: docker-compose
+      config:
+        computedata:
+          image: {get_param: DockerComputeDataImage}
+          container_name: computedata
+
+        libvirt:
+          image: {get_param: DockerLibvirtImage}
+          container_name: libvirt
+          net: host
+          pid: host
+          privileged: true
+          restart: always
+          volumes:
+           - /run:/run
+           - /lib/modules:/lib/modules:ro
+           - /var/lib/etc-data/libvirt/libvirtd.conf:/opt/kolla/libvirtd/libvirtd.conf
+           - /var/lib/nova/instances:/var/lib/nova/instances
+          environment:
+           - KOLLA_CONFIG_STRATEGY=CONFIG_EXTERNAL_COPY_ALWAYS
+          volumes_from:
+           - computedata
+
+  NovaComputeContainersDeployment:
+    type: OS::Heat::StructuredDeployments
+    depends_on: [CopyEtcDeployment, ComputePuppetDeployment, NovaComputeContainersDeploymentNetconfig, LibvirtContainersDeployment]
+    properties:
+      config: {get_resource: NovaComputeContainersConfig}
+      servers: {get_param: servers}
+
+  NovaComputeContainersConfig:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: docker-compose
+      config:
+        openvswitch:
+          image: {get_param: DockerOpenvswitchImage}
+          container_name: openvswitch
+          net: host
+          privileged: true
+          restart: always
+          volumes:
+           - /run:/run
+           - /lib/modules:/lib/modules:ro
+           - /var/lib/etc-data/neutron/neutron.conf:/etc/kolla/neutron-openvswitch-agent/:ro
+           - /var/lib/etc-data/neutron/plugins/ml2/ml2_conf.ini:/etc/kolla/neutron-openvswitch-agent/:ro
+          environment:
+           - KOLLA_CONFIG_STRATEGY=CONFIG_EXTERNAL_COPY_ALWAYS
+          volumes_from:
+           - computedata
+           # FIXME: Kolla now uses a JSON model to run custom commands.  We rebuilt a custom container to read in KOLLA_COMMAND_ARGS
+
+        # FIXME: Here we're subjugating kolla's start scripts because we want our custom run command
+        neutronagent:
+          image: {get_param: DockerOpenvswitchImage}
+          container_name: neutronagent
+          net: host
+          pid: host
+          privileged: true
+          restart: always
+          volumes:
+           - /run:/run
+           - /lib/modules:/lib/modules:ro
+           - /var/lib/etc-data/neutron/neutron.conf:/etc/neutron/neutron.conf:ro
+           - /var/lib/etc-data/neutron/plugins/ml2/openvswitch_agent.ini:/etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini:ro
+          environment:
+           - KOLLA_CONFIG_STRATEGY=CONFIG_EXTERNAL_COPY_ALWAYS
+           # FIXME: Kolla now uses a JSON model to run custom commands.  We rebuilt a custom container to read in KOLLA_COMMAND_ARGS
+           - KOLLA_COMMAND_ARGS=--config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini
+          volumes_from:
+           - computedata
+
+        novacompute:
+          image: {get_param: DockerComputeImage}
+          container_name: novacompute
+          net: host
+          privileged: true
+          restart: always
+          volumes:
+           - /run:/run
+           - /sys/fs/cgroup:/sys/fs/cgroup
+           - /lib/modules:/lib/modules:ro
+           - /var/lib/etc-data/:/etc/:ro
+           - /var/lib/nova/instances:/var/lib/nova/instances
+          volumes_from:
+           - computedata
+          # FIXME: this skips the kolla start.sh script and just starts Nova
+          # Ideally we'd have an environment that switched the kolla container
+          # to be externally configured.
+          command: /usr/bin/nova-compute
+
+  ExtraConfig:
+    depends_on: NovaComputeContainersDeployment
+    type: OS::TripleO::NodeExtraConfigPost
+    properties:
+        servers: {get_param: servers}

docker/copy-etc.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "Copying agent container /etc to /var/lib/etc-data"
+cp -a /etc/* /var/lib/etc-data/

docker/firstboot/install_docker_agents.yaml

@@ -0,0 +1,28 @@
+heat_template_version: 2014-10-16
+
+parameters:
+  DockerAgentImage:
+    type: string
+    default: dprince/heat-docker-agents-centos
+
+resources:
+
+  userdata:
+    type: OS::Heat::MultipartMime
+    properties:
+      parts:
+      - config: {get_resource: install_docker_agents}
+
+  install_docker_agents:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        str_replace:
+          params:
+            $agent_image: {get_param: DockerAgentImage}
+          template: {get_file: ./start_docker_agents.sh}
+
+outputs:
+  OS::stack_id:
+    value: {get_resource: userdata}

docker/firstboot/start_docker_agents.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+set -eux
+
+# firstboot isn't split out by role yet so we handle it this way
+if ! hostname | grep compute &>/dev/null; then
+ echo "Exiting. This script is only for the compute role."
+ exit 0
+fi
+
+mkdir -p /var/lib/etc-data/ #FIXME: this should be a docker data container
+
+# heat-docker-agents service
+cat <<EOF > /etc/systemd/system/heat-docker-agents.service
+
+[Unit]
+Description=Heat Docker Agent Container
+After=docker.service
+Requires=docker.service
+
+[Service]
+User=root
+Restart=on-failure
+ExecStartPre=-/usr/bin/docker kill heat-agents
+ExecStartPre=-/usr/bin/docker rm heat-agents
+ExecStartPre=/usr/bin/docker pull $agent_image
+ExecStart=/usr/bin/docker run --name heat-agents --privileged --net=host -v /var/lib/etc-data:/var/lib/etc-data -v /run:/run -v /etc:/host/etc -v /usr/bin/atomic:/usr/bin/atomic -v /var/lib/dhclient:/var/lib/dhclient -v /var/lib/cloud:/var/lib/cloud -v /var/lib/heat-cfntools:/var/lib/heat-cfntools --entrypoint=/usr/bin/os-collect-config $agent_image
+ExecStop=/usr/bin/docker stop heat-agents
+
+[Install]
+WantedBy=multi-user.target
+
+EOF
+
+# update docker for local insecure registry(optional)
+# Note: This is different for different docker versions
+# For older docker versions < 1.4.x use commented line
+#echo "OPTIONS='--insecure-registry $docker_registry'" >> /etc/sysconfig/docker
+#echo "ADD_REGISTRY='--registry-mirror $docker_registry'" >> /etc/sysconfig/docker
+
+# Local docker registry 1.8
+#/bin/sed -i s/ADD_REGISTRY/#ADD_REGISTRY/ /etc/sysconfig/docker
+
+/sbin/setenforce 0
+/sbin/modprobe ebtables
+
+# Create /var/lib/etc-data for now.  FIXME: This should go into a data container.
+#mkdir -p /var/lib/etc-data
+
+echo nameserver 8.8.8.8 > /etc/resolv.conf
+
+# We need hostname -f to return in a centos container for the puppet hook
+HOSTNAME=$(hostname)
+echo "127.0.0.1 $HOSTNAME.localdomain $HOSTNAME" >> /etc/hosts
+
+# Another hack.. we need latest docker..
+/usr/bin/systemctl stop docker.service
+/bin/curl -o /tmp/docker https://get.docker.com/builds/Linux/x86_64/docker-latest
+/bin/mount -o remount,rw /usr
+/bin/rm /bin/docker
+/bin/cp /tmp/docker /bin/docker
+/bin/chmod 755 /bin/docker
+
+# enable and start docker
+/usr/bin/systemctl enable docker.service
+/usr/bin/systemctl restart --no-block docker.service
+
+# enable and start heat-docker-agents
+chmod 0640 /etc/systemd/system/heat-docker-agents.service
+/usr/bin/systemctl enable heat-docker-agents.service
+/usr/bin/systemctl start --no-block heat-docker-agents.service
+
+# Disable NetworkManager and let the ifup/down scripts work properly.
+/usr/bin/systemctl disable NetworkManager
+/usr/bin/systemctl stop NetworkManager

environments/docker-rdo.yaml

@@ -0,0 +1,17 @@
+resource_registry:
+  # Docker container with heat agents for containerized compute node.
+  OS::TripleO::ComputePostDeployment: ../docker/compute-post.yaml
+  OS::TripleO::NodeUserData: ../docker/firstboot/install_docker_agents.yaml
+  OS::TripleO::Compute::Net::SoftwareConfig: ../net-config-bridge.yaml
+
+parameters:
+  NovaImage: fedora-atomic
+
+parameter_defaults:
+  DockerComputeImage: rthallisey/centos-binary-nova-compute:liberty
+  DockerComputeDataImage: kollaglue/centos-rdo-nova-compute-data:liberty2
+  DockerLibvirtImage: kollaglue/centos-rdo-nova-libvirt:liberty2
+  DockerNeutronAgentImage: kollaglue/centos-rdo-neutron-agents:liberty2
+  DockerOpenvswitchImage: rthallisey/centos-rdo-neutron-openvswitch-agent:latest
+  DockerOvsVswitchdImage: kollaglue/centos-rdo-ovs-vswitchd:liberty2
+  DockerOpenvswitchDBImage: kollaglue/centos-rdo-ovs-db-server:liberty2