From 1b699279a4bf9e87170fde3a4d89169bc558cfe0 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Fri, 9 Apr 2021 15:09:22 -0700 Subject: [PATCH] Run a git server in k8s for functional tests To make the functional tests easier to run locally, run a git server in k8s instead of on the local machine. Also, update the secret creation to work with kind as well as minikube. Change-Id: I4c00129c7105405b0f6256e769486e79c0b173fe --- .../zuul-operator-functional/pre-k8s.yaml | 8 ++ .../tasks/create_config.yaml | 17 ++-- .../tasks/create_test_secrets.yaml | 24 ++---- .../tasks/git_setup.yaml | 77 +++++++++++++------ .../tasks/zuul_web_check.yaml | 2 +- playbooks/zuul-operator-functional/test.yaml | 44 ++++++----- zuul_operator/templates/zuul.yaml | 2 +- 7 files changed, 101 insertions(+), 73 deletions(-) diff --git a/playbooks/zuul-operator-functional/pre-k8s.yaml b/playbooks/zuul-operator-functional/pre-k8s.yaml index e65993d..84c626c 100644 --- a/playbooks/zuul-operator-functional/pre-k8s.yaml +++ b/playbooks/zuul-operator-functional/pre-k8s.yaml @@ -9,11 +9,19 @@ minikube_dns_resolvers: - '1.1.1.1' - '8.8.8.8' + ensure_kubernetes_minikube_addons: + - ingress - role: use-buildset-registry buildset_registry_docker_user: root post_tasks: - name: Install openshift client for k8s tasks command: python3 -m pip install --user openshift + - name: Install websocket + become: true + package: + name: + - python3-websocket + - jq - name: check kubernetes connection command: timeout 10s kubectl get pods register: _api_ready diff --git a/playbooks/zuul-operator-functional/tasks/create_config.yaml b/playbooks/zuul-operator-functional/tasks/create_config.yaml index 6dad442..f9cd00b 100644 --- a/playbooks/zuul-operator-functional/tasks/create_config.yaml +++ b/playbooks/zuul-operator-functional/tasks/create_config.yaml @@ -1,13 +1,11 @@ - name: create config directory - become: yes - file: - path: /srv/git/config/ - state: directory - mode: 0777 + command: "git clone git://{{ cluster_ip }}:{{ git_server_port }}/config" + args: + chdir: "{{ git_root.path }}" - name: copy config content copy: - dest: "/srv/git/config/{{ item.name }}" + dest: "{{ git_root.path }}/config/{{ item.name }}" content: "{{ item.content }}" loop: - name: zuul.yaml @@ -55,7 +53,10 @@ - name: commit config shell: | - cd /srv/git/config/ - test -d .git || git init . + set -x + git add *.yaml git commit -m 'Setup config project' || true + git push --set-upstream git://{{ cluster_ip }}:{{ git_server_port }}/config master + args: + chdir: "{{ git_root.path }}/config" diff --git a/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml b/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml index 8142730..292685f 100644 --- a/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml +++ b/playbooks/zuul-operator-functional/tasks/create_test_secrets.yaml @@ -8,24 +8,13 @@ command: cat id_rsa register: _ssh_key -- name: Read generated kubectl configuration - command: | - sed -e 's#/home/zuul/.minikube/profiles/minikube/#/etc/kubernetes/#g' - -e 's#/home/zuul/.minikube/#/etc/kubernetes/#g' - ~/.kube/config +- name: Read kube server address + command: kubectl config view -o jsonpath='{.clusters[0].cluster.server}' register: _kube_config -- name: Read client cert - command: cat ~/.minikube/profiles/minikube/client.crt - register: _kube_cert - -- name: Read client key - command: cat ~/.minikube/profiles/minikube/client.key - register: _kube_key - -- name: Read ca cert - command: cat ~/.minikube/ca.crt - register: _kube_ca +- name: Read generated kubectl configuration + shell: "kubectl config view --flatten | sed -e 's#{{ _kube_config.stdout }}#https://kubernetes/#g'" + register: _kube_config - name: Read current context name command: kubectl config current-context @@ -79,6 +68,3 @@ - name: nodepool-kube-config data: kube.config: "{{ _kube_config.stdout }}" - client.crt: "{{ _kube_cert.stdout }}" - client.key: "{{ _kube_key.stdout }}" - ca.crt: "{{ _kube_ca.stdout }}" diff --git a/playbooks/zuul-operator-functional/tasks/git_setup.yaml b/playbooks/zuul-operator-functional/tasks/git_setup.yaml index c4e5419..a35baec 100644 --- a/playbooks/zuul-operator-functional/tasks/git_setup.yaml +++ b/playbooks/zuul-operator-functional/tasks/git_setup.yaml @@ -1,34 +1,61 @@ - - name: setup gitconfig shell: | if ! test -f ~/.gitconfig && ! test -d ~/.config/git ; then git config --global user.email "you@example.com" git config --global user.name "Your Name" fi - sudo mkdir -p /srv/git -- name: setup local-git service - become: yes - copy: - dest: /etc/systemd/system/local-git.service - content: | - [Unit] - Description=local git service - After=syslogp.target network.target +- k8s: + namespace: default + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: git-server + spec: + replicas: 1 + selector: + matchLabels: + app: git-server + template: + metadata: + labels: + app: git-server + spec: + volumes: + - name: git-repos + emptyDir: {} + containers: + - name: git-server + image: docker.io/sdelrio/git-daemon:latest + ports: + - containerPort: 9418 + volumeMounts: + - name: git-repos + mountPath: /git + initContainers: + - name: init-config + image: docker.io/sdelrio/git-daemon:latest + command: ['/usr/bin/git', 'init', '--bare', '/git/config'] + volumeMounts: + - name: git-repos + mountPath: /git - [Service] - Type=simple - SyslogIdentifier=local-git - ExecStart=/usr/bin/git daemon --export-all --reuseaddr --verbose --base-path=/srv/git/ /srv/git/ - Restart=always - RestartSec=3s +- k8s: + namespace: default + definition: + apiVersion: v1 + kind: Service + metadata: + name: git-server + labels: + app: git-server + spec: + type: NodePort + ports: + - port: 9418 + selector: + app: git-server - [Install] - WantedBy=multi-user.target - -- name: start local-git service - become: yes - systemd: - daemon_reload: true - name: "local-git" - state: started +- name: Wait for scheduler deployment + command: timeout 10m kubectl rollout status deployment/git-server diff --git a/playbooks/zuul-operator-functional/tasks/zuul_web_check.yaml b/playbooks/zuul-operator-functional/tasks/zuul_web_check.yaml index 3a189c4..830276c 100644 --- a/playbooks/zuul-operator-functional/tasks/zuul_web_check.yaml +++ b/playbooks/zuul-operator-functional/tasks/zuul_web_check.yaml @@ -2,6 +2,6 @@ uri: url: "{{ zuul_web_url }}{{ endpoint }}" register: result - until: "result.json is defined and result.json{% if expected is defined %} and result.json == expected{% endif %}" + until: "result.json is defined and result.json and (expected is not defined or result.json == expected)" retries: 600 delay: 1 diff --git a/playbooks/zuul-operator-functional/test.yaml b/playbooks/zuul-operator-functional/test.yaml index 8986a44..684db89 100644 --- a/playbooks/zuul-operator-functional/test.yaml +++ b/playbooks/zuul-operator-functional/test.yaml @@ -1,26 +1,24 @@ - name: run functional tst hosts: all - pre_tasks: - - name: install git - become: yes - package: - name: - - git - - jq - - - name: install websocket client - become: yes - command: python3 -m pip install websocket-client - tasks: - - name: get rest api url - command: kubectl get svc zuul-web -o jsonpath='{.spec.clusterIP}' - register: zuul_web_ip + - name: Create temp git root directory + tempfile: + state: directory + register: git_root + + - name: get cluster ip + command: /tmp/minikube ip + register: minikube_ip + failed_when: false + + - name: set cluster ip + set_fact: + cluster_ip: "{{ cluster_ip | default(minikube_ip.stdout_lines[0]) }}" - name: set fact zuul_web_url set_fact: - zuul_web_url: "http://{{ zuul_web_ip.stdout_lines[0] }}:9000" - zuul_ws_url: "ws://{{ zuul_web_ip.stdout_lines[0] }}:9000" + zuul_web_url: "http://{{ cluster_ip }}" + zuul_ws_url: "ws://{{ cluster_ip }}" - name: ensure zuul web api is working when: skip_check is not defined @@ -35,6 +33,14 @@ - name: setup git service include_tasks: tasks/git_setup.yaml + - name: get git-server port + command: kubectl get svc git-server -o jsonpath='{.spec.ports[0].nodePort}' + register: git_server_port + + - name: set git-server port + set_fact: + git_server_port: '{{ git_server_port.stdout_lines[0] }}' + - name: create a config project include_tasks: tasks/create_config.yaml @@ -83,7 +89,7 @@ baseurl: https://opendev.org driver: git local-git: - baseurl: "git://{{ ansible_all_ipv4_addresses[0] }}/" + baseurl: "git://git-server.default/" driver: git externalConfig: kubernetes: @@ -131,7 +137,7 @@ - name: connect to console-stream command: | - python3 /usr/local/bin/wsdump.py -r --eof-wait 5 -t '{"uuid":"{{ _job_uuid.stdout_lines[0] }}","logfile":"console.log"}' {{ zuul_ws_url }}/api/tenant/local/console-stream + wsdump -r --eof-wait 5 -t '{"uuid":"{{ _job_uuid.stdout_lines[0] }}","logfile":"console.log"}' {{ zuul_ws_url }}/api/tenant/local/console-stream register: console_stream - name: show console stream diff --git a/zuul_operator/templates/zuul.yaml b/zuul_operator/templates/zuul.yaml index 3f7bd98..6736ae3 100644 --- a/zuul_operator/templates/zuul.yaml +++ b/zuul_operator/templates/zuul.yaml @@ -76,7 +76,7 @@ metadata: app.kubernetes.io/part-of: zuul app.kubernetes.io/component: zuul-web spec: - #type: NodePort + type: NodePort ports: - name: zuul-web port: 9000