From 97aa27080f1797e40e8ef959b779c78e28b20d6d Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 7 Jul 2021 10:29:46 -0700 Subject: [PATCH] Add static node to functional test So that we can better test functions related to ssh connectivity, add a static node to the functional test. Change-Id: I3dfdaca0b4e5dbfb35e4a645c7bcb56e0eec88c2 --- playbooks/zuul-operator-functional/run.yaml | 4 + .../static-node/Dockerfile | 29 ++++++++ .../static-node/ssh_host_ed25519_key | 7 ++ .../static-node/ssh_host_ed25519_key.pub | 1 + .../tasks/create_config.yaml | 42 +++++++++-- .../tasks/create_static_node.yaml | 74 +++++++++++++++++++ .../tasks/create_test_secrets.yaml | 20 +++++ .../tasks/wait_services.yaml | 9 ++- playbooks/zuul-operator-functional/test.yaml | 31 +++++++- tools/vars.yaml | 3 +- 10 files changed, 207 insertions(+), 13 deletions(-) create mode 100644 playbooks/zuul-operator-functional/static-node/Dockerfile create mode 100644 playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key create mode 100644 playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key.pub create mode 100644 playbooks/zuul-operator-functional/tasks/create_static_node.yaml diff --git a/playbooks/zuul-operator-functional/run.yaml b/playbooks/zuul-operator-functional/run.yaml index fe6f0fa..2fc64e4 100644 --- a/playbooks/zuul-operator-functional/run.yaml +++ b/playbooks/zuul-operator-functional/run.yaml @@ -5,6 +5,7 @@ # locally for development. install_operator: true zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul-operator'].src_dir }}" + runtime: minikube tasks: - name: Setup CRD command: kubectl apply -f deploy/crds/zuul-ci_v1alpha2_zuul_crd.yaml -f deploy/rbac.yaml @@ -17,6 +18,9 @@ args: chdir: "{{ zuul_work_dir }}" + - name: Create static node + include_tasks: ./tasks/create_static_node.yaml + - name: Create required secret include_tasks: ./tasks/create_test_secrets.yaml diff --git a/playbooks/zuul-operator-functional/static-node/Dockerfile b/playbooks/zuul-operator-functional/static-node/Dockerfile new file mode 100644 index 0000000..90c2d75 --- /dev/null +++ b/playbooks/zuul-operator-functional/static-node/Dockerfile @@ -0,0 +1,29 @@ +FROM ubuntu:20.04 + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -y install \ + git \ + openssh-server \ + rsync \ + && rm -rf /var/lib/apt/lists/* + +# If proxy environment variables supplied during build allow pass through via +# ~/.ssh/environment, as the test env likely will require these set for any +# job launched on the node to be capable of internet access. +RUN set -e ; \ + mkdir /var/run/sshd ; \ + mkdir -p -m 0700 ~/.ssh; \ + if [ -n "${http_proxy}" ]; then \ + sed -ri 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config; \ + echo "http_proxy=${http_proxy}" > ~/.ssh/environment; \ + echo "https_proxy=${https_proxy}" >> ~/.ssh/environment; \ + echo "no_proxy=${no_proxy}" >> ~/.ssh/environment; \ + fi \ + ; + +COPY --chown=root:root ./ssh_host_ed25519_key /etc/ssh/ssh_host_ed25519_key +RUN chmod 0600 /etc/ssh/ssh_host_ed25519_key + +EXPOSE 22 + +ENTRYPOINT ["/usr/sbin/sshd", "-D" ] diff --git a/playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key b/playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key new file mode 100644 index 0000000..68ce704 --- /dev/null +++ b/playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACDoByWHoyDSCs1IrkidjDfLj3PnXxa2fSR9YS1kvICPoAAAAJiZWqEimVqh +IgAAAAtzc2gtZWQyNTUxOQAAACDoByWHoyDSCs1IrkidjDfLj3PnXxa2fSR9YS1kvICPoA +AAAEA9aXkLh3eloH1HMQ2RR3DQ2bzIMROVxkvKKDmeYsDlLegHJYejINIKzUiuSJ2MN8uP +c+dfFrZ9JH1hLWS8gI+gAAAAEXJvb3RAMzI5NmRjMDg4ODQ2AQIDBA== +-----END OPENSSH PRIVATE KEY----- diff --git a/playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key.pub b/playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key.pub new file mode 100644 index 0000000..a95c202 --- /dev/null +++ b/playbooks/zuul-operator-functional/static-node/ssh_host_ed25519_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOgHJYejINIKzUiuSJ2MN8uPc+dfFrZ9JH1hLWS8gI+g root@3296dc088846 diff --git a/playbooks/zuul-operator-functional/tasks/create_config.yaml b/playbooks/zuul-operator-functional/tasks/create_config.yaml index f9cd00b..46bb67a 100644 --- a/playbooks/zuul-operator-functional/tasks/create_config.yaml +++ b/playbooks/zuul-operator-functional/tasks/create_config.yaml @@ -23,32 +23,60 @@ - name: container label: pod-fedora-32 + - nodeset: + name: ubuntu-focal + nodes: + - name: ubuntu-focal + label: ubuntu-focal + - job: - name: test-job - run: test.yaml + name: test-pod + run: test-pod.yaml # skip zuul-base-jobs for now as it doesn't work with kubectl roles: - zuul: zuul/zuul-jobs parent: null nodeset: pod-fedora + - job: + name: test-static-node + run: test-static-node.yaml + roles: + - zuul: zuul/zuul-jobs + parent: null + nodeset: ubuntu-focal + - project: periodic: jobs: - - test-job + - test-pod + - test-static-node - - name: test.yaml + - name: test-pod.yaml content: | - hosts: localhost tasks: - - include_role: - name: start-zuul-console - - debug: msg='Demo job is running' - name: check job volume stat: path: /system-dbus register: _job_volume failed_when: not _job_volume.stat.isdir + - hosts: container + tasks: + - include_role: + name: start-zuul-console + - debug: msg='Demo job is running on container' + - shell: id + - pause: seconds=30 + + - name: test-static-node.yaml + content: | + - hosts: all + tasks: + - include_role: + name: start-zuul-console + - debug: msg='Demo job is running on static node' + - shell: id - pause: seconds=30 - name: commit config diff --git a/playbooks/zuul-operator-functional/tasks/create_static_node.yaml b/playbooks/zuul-operator-functional/tasks/create_static_node.yaml new file mode 100644 index 0000000..db95854 --- /dev/null +++ b/playbooks/zuul-operator-functional/tasks/create_static_node.yaml @@ -0,0 +1,74 @@ +- name: Create static node image + when: "runtime == 'minikube'" + args: + chdir: "{{ zuul_work_dir }}/playbooks/zuul-operator-functional/static-node" + shell: | + /tmp/minikube image build . -t static-node + +- name: Create static node image + when: "runtime == 'kind'" + args: + chdir: "{{ zuul_work_dir }}/playbooks/zuul-operator-functional/static-node" + shell: | + docker build . -t static-node + kind load docker-image static-node + +- name: Run static node + k8s: + namespace: "default" + definition: + apiVersion: v1 + kind: Pod + metadata: + name: node + labels: + app: static-node + spec: + containers: + - name: node + image: static-node + imagePullPolicy: Never + lifecycle: + postStart: + exec: + command: + - bash + - -c + - mkdir -p /root/.ssh && chmod 0600 /root/.ssh && cp /sshkeys/authorized_keys /root/.ssh + ports: + - name: ssh + containerPort: 22 + protocol: TCP + - name: zuul + containerPort: 19885 + protocol: TCP + volumeMounts: + - name: authorized-keys + mountPath: /sshkeys + readOnly: true + volumes: + - name: authorized-keys + secret: + secretName: static-node-authorized-keys + defaultMode: 0600 + +- name: Create static node service + k8s: + namespace: "default" + definition: + apiVersion: v1 + kind: Service + metadata: + name: node + spec: + selector: + app: static-node + ports: + - name: ssh + port: 22 + targetPort: 22 + protocol: TCP + - name: zuul + port: 19885 + targetPort: 19885 + protocol: TCP diff --git a/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml b/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml index 45d9d25..98342e9 100644 --- a/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml +++ b/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml @@ -8,6 +8,10 @@ command: cat id_rsa register: _ssh_key +- name: Read generated ssh public key + command: cat id_rsa.pub + register: _ssh_key_pub + - name: Read kube server address command: kubectl config view -o jsonpath='{.clusters[0].cluster.server}' register: _kube_config @@ -30,6 +34,10 @@ name: "{{ item.name }}" stringData: "{{ item.data }}" loop: + - name: static-node-authorized-keys + data: + authorized_keys: "{{ _ssh_key_pub.stdout }}" + - name: executor-ssh-key data: sshkey: "{{ _ssh_key.stdout }}" @@ -52,6 +60,8 @@ labels: - name: pod-fedora-32 min-ready: 1 + - name: ubuntu-focal + min-ready: 1 providers: - name: kube-cluster driver: kubernetes @@ -64,6 +74,16 @@ type: pod image: docker.io/fedora:32 python-path: /bin/python3 + - name: static-vms + driver: static + pools: + - name: main + nodes: + - name: node.default + labels: ubuntu-focal + host-key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOgHJYejINIKzUiuSJ2MN8uPc+dfFrZ9JH1hLWS8gI+g" + python-path: /usr/bin/python3 + username: root - name: nodepool-kube-config data: diff --git a/playbooks/zuul-operator-functional/tasks/wait_services.yaml b/playbooks/zuul-operator-functional/tasks/wait_services.yaml index 10d3043..0da89f1 100644 --- a/playbooks/zuul-operator-functional/tasks/wait_services.yaml +++ b/playbooks/zuul-operator-functional/tasks/wait_services.yaml @@ -19,9 +19,16 @@ - name: Wait for executor deployment command: timeout 10m kubectl rollout status statefulset/zuul-executor -- name: Wait 8 minutes for launcher to settle +- name: Wait 8 minutes for kube launcher to settle command: kubectl logs deployment/nodepool-launcher-kube-cluster register: _launcher_log until: "'Starting PoolWorker' in _launcher_log.stdout" delay: 10 retries: 48 + +- name: Wait 8 minutes for static launcher to settle + command: kubectl logs deployment/nodepool-launcher-static-vms + register: _launcher_log + until: "'Starting PoolWorker' in _launcher_log.stdout" + delay: 10 + retries: 48 diff --git a/playbooks/zuul-operator-functional/test.yaml b/playbooks/zuul-operator-functional/test.yaml index d87ae45..46bb568 100644 --- a/playbooks/zuul-operator-functional/test.yaml +++ b/playbooks/zuul-operator-functional/test.yaml @@ -1,5 +1,8 @@ - name: run functional tst hosts: all + vars: + zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul-operator'].src_dir }}" + runtime: minikube tasks: - name: Install ingress include_tasks: tasks/ingress.yaml @@ -10,13 +13,24 @@ register: git_root - name: get cluster ip + when: runtime == 'minikube' command: /tmp/minikube ip - register: minikube_ip - failed_when: false + register: _cluster_ip - name: set cluster ip + when: runtime == 'minikube' set_fact: - cluster_ip: "{{ cluster_ip | default(minikube_ip.stdout_lines[0]) }}" + cluster_ip: "{{ _cluster_ip.stdout_lines[0] }}" + + - name: get cluster ip + when: runtime == 'kind' + command: docker inspect -f "{% raw %}{{ .NetworkSettings.IPAddress }}{% endraw %}" kind-control-plane + register: _cluster_ip + + - name: set cluster ip + when: runtime == 'kind' + set_fact: + cluster_ip: "{{ _cluster_ip.stdout_lines[0] }}" - name: set fact zuul_web_url set_fact: @@ -121,6 +135,14 @@ # 1 queue means a job is running queue: 1 + - name: get build results + uri: + url: "{{ zuul_web_url }}/api/tenant/local/builds?complete=true" + register: result + until: "result.json is defined and result.json and (result.json|length) > 1" + retries: 600 + delay: 1 + - name: get buillds results include_tasks: tasks/zuul_web_check.yaml vars: @@ -130,13 +152,14 @@ assert: that: - result.json[0].result == 'SUCCESS' + - result.json[1].result == 'SUCCESS' - name: grab job uuid shell: | curl -s {{ zuul_web_url }}/api/tenant/local/status | jq -r '.pipelines[].change_queues[].heads[][].jobs[].uuid' register: _job_uuid # Wait until the executor start the job - until: _job_uuid.stdout != "" and _job_uuid.stdout != "null" + until: _job_uuid.stdout != "" and "null" not in _job_uuid.stdout retries: 60 delay: 1 diff --git a/tools/vars.yaml b/tools/vars.yaml index fc6029f..a6fa51d 100644 --- a/tools/vars.yaml +++ b/tools/vars.yaml @@ -1,3 +1,4 @@ namespace: default -cluster_ip: 172.17.0.2 install_operator: false +zuul_work_dir: "{{ playbook_dir }}/../../" +runtime: kind