From bbf32935dc435cb8d6c074f76e41dd4056bc70ef Mon Sep 17 00:00:00 2001 From: portdirect Date: Sat, 30 Dec 2017 19:29:02 -0500 Subject: [PATCH] Docker: Run docker without iptables This PS updates the docker daemon settings to run without apply iptables rules. This simplifies host network management by removing one of the actors interacting with iptables. Change-Id: I335247afddf736b60212d199a3b860c3c792977f --- .../build-images/tasks/kubeadm-aio.yaml | 46 +++++++++++++------ .../playbooks/deploy-docker/tasks/main.yaml | 31 ++++++++----- .../templates/centos-docker.service.j2 | 3 +- .../templates/fedora-docker.service.j2 | 3 +- .../templates/ubuntu-docker.service.j2 | 30 ++++++++++++ .../templates/kubelet.service.j2 | 3 +- 6 files changed, 87 insertions(+), 29 deletions(-) create mode 100644 tools/gate/playbooks/deploy-docker/templates/ubuntu-docker.service.j2 diff --git a/tools/gate/playbooks/build-images/tasks/kubeadm-aio.yaml b/tools/gate/playbooks/build-images/tasks/kubeadm-aio.yaml index b6b0f94390..d56c54bcae 100644 --- a/tools/gate/playbooks/build-images/tasks/kubeadm-aio.yaml +++ b/tools/gate/playbooks/build-images/tasks/kubeadm-aio.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +#NOTE(portdirect): Untill https://github.com/ansible/ansible/issues/21433 is +# reolved, we build with a shell script to make use of the host network. - name: Kubeadm-AIO build block: #NOTE(portdirect): we do this to ensure we are feeding the docker build @@ -19,17 +21,33 @@ - name: Kubeadm-AIO image build path shell: cd "{{ work_dir }}"; pwd register: kubeadm_aio_path - - name: build the Kubeadm-AIO image - docker_image: - path: "{{ kubeadm_aio_path.stdout }}/" - name: "{{ images.kubernetes.kubeadm_aio }}" - dockerfile: "tools/images/kubeadm-aio/Dockerfile" - force: yes - pull: yes - state: present - rm: yes - buildargs: - KUBE_VERSION: "{{ version.kubernetes }}" - CNI_VERSION: "{{ version.cni }}" - HELM_VERSION: "{{ version.helm }}" - CHARTS: "calico,flannel,tiller,kube-dns" + # - name: build the Kubeadm-AIO image + # docker_image: + # path: "{{ kubeadm_aio_path.stdout }}/" + # name: "{{ images.kubernetes.kubeadm_aio }}" + # dockerfile: "tools/images/kubeadm-aio/Dockerfile" + # force: yes + # pull: yes + # state: present + # rm: yes + # buildargs: + # KUBE_VERSION: "{{ version.kubernetes }}" + # CNI_VERSION: "{{ version.cni }}" + # HELM_VERSION: "{{ version.helm }}" + # CHARTS: "calico,flannel,tiller,kube-dns" + - name: Kubeadm-AIO image build path + shell: |- + set -e + docker build \ + --network host \ + --force-rm \ + --tag "{{ images.kubernetes.kubeadm_aio }}" \ + --file tools/images/kubeadm-aio/Dockerfile \ + --build-arg KUBE_VERSION="{{ version.kubernetes }}" \ + --build-arg CNI_VERSION="{{ version.cni }}" \ + --build-arg HELM_VERSION="{{ version.helm }}" \ + --build-arg CHARTS="calico,flannel,tiller,kube-dns" \ + . + args: + chdir: "{{ kubeadm_aio_path.stdout }}/" + executable: /bin/bash diff --git a/tools/gate/playbooks/deploy-docker/tasks/main.yaml b/tools/gate/playbooks/deploy-docker/tasks/main.yaml index 97ac3a797b..dc8d27c901 100644 --- a/tools/gate/playbooks/deploy-docker/tasks/main.yaml +++ b/tools/gate/playbooks/deploy-docker/tasks/main.yaml @@ -17,18 +17,6 @@ register: need_docker ignore_errors: True -- name: deploy docker packages - when: need_docker | failed - include_role: - name: deploy-package - tasks_from: dist - vars: - packages: - deb: - - docker.io - rpm: - - docker-latest - - name: centos | moving systemd unit into place when: ( ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' ) and ( need_docker | failed ) template: @@ -43,6 +31,25 @@ dest: /etc/systemd/system/docker.service mode: 0640 +- name: ubuntu | moving systemd unit into place + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' ) and ( need_docker | failed ) + template: + src: ubuntu-docker.service.j2 + dest: /etc/systemd/system/docker.service + mode: 0640 + +- name: deploy docker packages + when: need_docker | failed + include_role: + name: deploy-package + tasks_from: dist + vars: + packages: + deb: + - docker.io + rpm: + - docker-latest + - name: restarting docker systemd: state: restarted diff --git a/tools/gate/playbooks/deploy-docker/templates/centos-docker.service.j2 b/tools/gate/playbooks/deploy-docker/templates/centos-docker.service.j2 index 5298225e65..dfac46188e 100644 --- a/tools/gate/playbooks/deploy-docker/templates/centos-docker.service.j2 +++ b/tools/gate/playbooks/deploy-docker/templates/centos-docker.service.j2 @@ -17,7 +17,8 @@ ExecStart=/usr/bin/dockerd-latest \ --userland-proxy-path=/usr/libexec/docker/docker-proxy-latest \ -g /var/lib/docker \ --storage-driver=overlay \ - --log-driver=json-file + --log-driver=json-file \ + --iptables=false ExecReload=/bin/kill -s HUP $MAINPID LimitNOFILE=1048576 LimitNPROC=1048576 diff --git a/tools/gate/playbooks/deploy-docker/templates/fedora-docker.service.j2 b/tools/gate/playbooks/deploy-docker/templates/fedora-docker.service.j2 index 4e7e763e2a..c6ba16b7d7 100644 --- a/tools/gate/playbooks/deploy-docker/templates/fedora-docker.service.j2 +++ b/tools/gate/playbooks/deploy-docker/templates/fedora-docker.service.j2 @@ -16,7 +16,8 @@ ExecStart=/usr/bin/dockerd-latest \ --userland-proxy-path=/usr/libexec/docker/docker-proxy-latest \ -g /var/lib/docker \ --storage-driver=overlay2 \ - --log-driver=json-file + --log-driver=json-file \ + --iptables=false ExecReload=/bin/kill -s HUP $MAINPID TasksMax=8192 LimitNOFILE=1048576 diff --git a/tools/gate/playbooks/deploy-docker/templates/ubuntu-docker.service.j2 b/tools/gate/playbooks/deploy-docker/templates/ubuntu-docker.service.j2 new file mode 100644 index 0000000000..2451b19803 --- /dev/null +++ b/tools/gate/playbooks/deploy-docker/templates/ubuntu-docker.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +After=network.target docker.socket firewalld.service +Requires=docker.socket + +[Service] +Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker +EnvironmentFile=-/etc/default/docker +ExecStart=/usr/bin/dockerd --iptables=false -H fd:// $DOCKER_OPTS +ExecReload=/bin/kill -s HUP $MAINPID +LimitNOFILE=1048576 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +# Uncomment TasksMax if your systemd version supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +TimeoutStartSec=0 +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes +# kill only the docker process, not all processes in the cgroup +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/kubelet.service.j2 b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/kubelet.service.j2 index 62a4e77409..46fcdd467c 100644 --- a/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/kubelet.service.j2 +++ b/tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubelet/templates/kubelet.service.j2 @@ -4,7 +4,8 @@ Documentation=http://kubernetes.io/docs/ [Service] ExecStartPre=/sbin/swapoff -a -ExecStartPre=/bin/bash -c "echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables" +ExecStartPre=/bin/bash -cex "modprobe br_netfilter" +ExecStartPre=/bin/bash -cex "echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables" ExecStart=/usr/bin/kubelet Restart=always StartLimitInterval=0