From fec27296c8ff488f1889e1bf854f8f5668e57681 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 27 Mar 2023 16:34:59 +1100 Subject: [PATCH] remove-registry-tag: role to delete tags from registry This is a role to abstract removal of tags from registries, which is an operation that practically has to be done via the registry API. This implements removing tags from the quay and docker API's. For the common case of working with a repository like "quay.io/org/project" there is minimal configuration. However, if you run a private repository, this is flexible with a few extra variables to tell the role to use the quay API but your own URL. By default it clears out old tags from the Zuul promote pipeline. However if you set registry_tag_remove_tag it will only remove that one tag. This is inspired by the current work done in promote-docker-image role. Change-Id: I7f2d9d00024e34451e2d20b2c2f8171ecd151943 --- doc/source/container-roles.rst | 1 + roles/remove-registry-tag/README.rst | 92 ++++++++++++++++++++ roles/remove-registry-tag/defaults/main.yaml | 2 + roles/remove-registry-tag/tasks/docker.yaml | 69 +++++++++++++++ roles/remove-registry-tag/tasks/main.yaml | 27 ++++++ roles/remove-registry-tag/tasks/quay.yaml | 55 ++++++++++++ 6 files changed, 246 insertions(+) create mode 100644 roles/remove-registry-tag/README.rst create mode 100644 roles/remove-registry-tag/defaults/main.yaml create mode 100644 roles/remove-registry-tag/tasks/docker.yaml create mode 100644 roles/remove-registry-tag/tasks/main.yaml create mode 100644 roles/remove-registry-tag/tasks/quay.yaml diff --git a/doc/source/container-roles.rst b/doc/source/container-roles.rst index 24a254793..8d16c7c24 100644 --- a/doc/source/container-roles.rst +++ b/doc/source/container-roles.rst @@ -15,6 +15,7 @@ Container Roles .. zuul:autorole:: promote-docker-image .. zuul:autorole:: pull-from-intermediate-registry .. zuul:autorole:: push-to-intermediate-registry +.. zuul:autorole:: remove-registry-tag .. zuul:autorole:: run-buildset-registry .. zuul:autorole:: upload-container-image .. zuul:autorole:: upload-docker-image diff --git a/roles/remove-registry-tag/README.rst b/roles/remove-registry-tag/README.rst new file mode 100644 index 000000000..c862d86c2 --- /dev/null +++ b/roles/remove-registry-tag/README.rst @@ -0,0 +1,92 @@ +Remove tags from registry + +This role creates a generic interface for removing tags from a +container registry. The OCI distribution API (implemented essentially +all registries) does specify a tag deletion endpoint, but as at +2023-03 essentially no registries implement it. This means +practically we must talk to the per-registry API directly to remove +tags. The methods to delete tags are generally similar across +registries, but differ slightly in endpoint names, etc. + +This role can run in two modes; either removing a single specific tag, +or it can run a cleanup process removing all tags that match a given +prefix and have not been modified in a given amount of time. + +For public registries this role should guess the API from the +repository name. If you are running against a private registry, you +will need to explicitly specify the API type and URL prefix to +communicate to using arguments below. + +**Role Variables** + +.. zuul:rolevar:: remove_registry_tag_repository + :type: string + + Required. This must be the full repository; + e.g. ``quay.io/organisation/image`` + +.. zuul:rolevar:: container_registry_credentials + :type: dict + + Required. This is expected to be a Zuul secret in dictionary form. + For convenience this is in the same format as the + ``container_registry_credentials`` variable used by the other + container roles. You must specify the correct variables for the + registry you are communicating with: + + * **quay.io** : Specify an ``api_key`` which is issued from an + application assigned to an organisation. See + ``__ + * **docker.io** : Username and password + + Example: + + .. code-block:: yaml + + container_registry_credentials: + quay.io: + api_token: 'abcd1234' + docker.io: + username: 'username' + password: 'password' + +.. zuul:rolevar:: remove_registry_tag_tag + :type: string + + Optional. If set, the specific tag to remove. + +.. zuul:rolevar:: remove_registry_tag_regex + :type: string + :default: '^change_.*$|^{{ zuul.pipeline }}_.*$' + + Optional. If + :zuul:rolevar:`remove-registry-tag.remove_registry_tag_tag` is + unset, any tags matching this regex *and* exceeding the age in + :zuul:rolevar:`remove-registry-tag.remove_registry_tag_age` will be + removed. The default is tags matching those created by the promote + upload roles. + +.. zuul:rolevar:: remove_registry_tag_age + :type: int + :default: 86400 + + Optional. The age, in seconds, a tag that matches + :zuul:rolevar:`remove-registry-tag.remove_registry_tag_regex` + last-modified timestamp must exceed to be removed. + +.. zuul:rolevar:: remove_registry_tag_api_type + :type: string + + Optional. By default the role will guess the API type from the + repository name. However, if you need to override this choice + specify one of: + + * quay + * docker + +.. zuul:rolevar:: remove_registry_tag_api_url + :type: string + + Optional. This role will use the default URL for the given + registry API. If you need to override this choice, specify this + variable. diff --git a/roles/remove-registry-tag/defaults/main.yaml b/roles/remove-registry-tag/defaults/main.yaml new file mode 100644 index 000000000..42d68de1c --- /dev/null +++ b/roles/remove-registry-tag/defaults/main.yaml @@ -0,0 +1,2 @@ +remove_registry_tag_regex: '^change_.*$|^{{ zuul.pipeline }}_.*$' +remove_registry_tag_age: 86400 diff --git a/roles/remove-registry-tag/tasks/docker.yaml b/roles/remove-registry-tag/tasks/docker.yaml new file mode 100644 index 000000000..7fc31bef4 --- /dev/null +++ b/roles/remove-registry-tag/tasks/docker.yaml @@ -0,0 +1,69 @@ +- name: Ensure registry token is set + assert: + that: > + (container_registry_credentials[_registry].username is defined) and + (container_registry_credentials[_registry].password is defined) + +- name: Set API base + when: remove_registry_tag_api_url is not defined + set_fact: + remove_registry_tag_api_url: 'https://hub.docker.com/v2' + +- name: Delete single tag + when: remove_registry_tag_tag is defined + set_fact: + _to_delete: + - '{{ remove_registry_tag_tag }}' + +- name: Iterate old tags + when: remove_registry_tag_tag is not defined + block: + - name: Setup vars + set_fact: + _to_delete: [] + + - name: Get project tags + uri: + url: '{{ remove_registry_tag_api_url }}/repositories/{{ _repopath }}/tags?page_size=1000' + status_code: 200 + register: _tags + + - name: Build list of old tags + loop: "{{ _tags.json.results }}" + loop_control: + loop_var: zj_docker_tag + set_fact: + _to_delete: '{{ _to_delete|default([]) + [zj_docker_tag] }}' + when: + - zj_docker_tag.name is regex(remove_registry_tag_regex) + # Was updated > 24 hours ago: + - "((ansible_date_time.iso8601 | regex_replace('^(....-..-..)T(..:..:..).*Z', '\\\\1 \\\\2') | to_datetime) - (zj_docker_tag.last_updated | regex_replace('^(....-..-..)T(..:..:..).*Z', '\\\\1 \\\\2') | to_datetime)).seconds > remove_registry_tag_age" + +- name: List tags to remove + debug: + var: _to_delete + +- name: Get dockerhub JWT token + no_log: true + uri: + url: "{{ remove_registry_tag_api_url }}/users/login/" + body_format: json + body: + username: "{{ container_registry_credentials[_registry].username }}" + password: "{{ container_registry_credentials[_registry].password }}" + register: jwt_token + delay: 5 + retries: 3 + until: jwt_token and jwt_token.status==200 + +- name: Delete tag + no_log: true + uri: + url: '{{ remove_registry_tag_api_url }}/repositories/{{ _repopath }}/tags/{{ zj_docker_tag }}' + method: DELETE + status_code: [200, 204] + headers: + 'Authorization': 'JWT {{ jwt_token.json.token }}' + loop: '{{ _to_delete }}' + loop_control: + loop_var: zj_docker_tag diff --git a/roles/remove-registry-tag/tasks/main.yaml b/roles/remove-registry-tag/tasks/main.yaml new file mode 100644 index 000000000..c76dec095 --- /dev/null +++ b/roles/remove-registry-tag/tasks/main.yaml @@ -0,0 +1,27 @@ +- name: Ensure repository is specified + assert: + that: remove_registry_tag_repository is defined + +- name: Validate remove_registry_tag_repository is full "url" + when: + - "'/' not in remove_registry_tag_repository" + fail: + msg: "{{ remove_registry_tag_repository }} must be a full container image url including registry location" + +- name: Parse out repo path from full "url" + set_fact: + _registry: "{{ (remove_registry_tag_repository | split('/', 1)).0 }}" + _repopath: "{{ (remove_registry_tag_repository | split('/', 1)).1 }}" + +- name: Autoprobe for quay.io + when: remove_registry_tag_api_type is not defined and "quay.io" in _registry + set_fact: + remove_registry_tag_api_type: "quay" + +- name: Autoprobe for docker + when: remove_registry_tag_api_type is not defined and "docker.io" in _registry + set_fact: + remove_registry_tag_api_type: "docker" + +- name: Remove tags + include_tasks: '{{ remove_registry_tag_api_type }}.yaml' diff --git a/roles/remove-registry-tag/tasks/quay.yaml b/roles/remove-registry-tag/tasks/quay.yaml new file mode 100644 index 000000000..7306ef641 --- /dev/null +++ b/roles/remove-registry-tag/tasks/quay.yaml @@ -0,0 +1,55 @@ +- name: Ensure registry token is set + assert: + that: container_registry_credentials[_registry].api_token is defined + no_log: true + +- name: Set API base + when: remove_registry_tag_api_url is not defined + set_fact: + remove_registry_tag_api_url: 'https://{{ _registry }}/api/v1' + +- name: Delete single tag + when: remove_registry_tag_tag is defined + set_fact: + _to_delete: + - '{{ remove_registry_tag_tag }}' + +- name: Iterate old tags + when: remove_registry_tag_tag is not defined + block: + - name: Setup vars + set_fact: + _to_delete: [] + + - name: Get project tags + uri: + url: '{{ remove_registry_tag_api_url }}/repository/{{ _repopath }}/tag/' + status_code: 200 + register: _tags + + - name: Build list of old tags + loop: "{{ _tags.json.tags }}" + loop_control: + loop_var: zj_quay_tag + set_fact: + _to_delete: '{{ _to_delete|default([]) + [zj_quay_tag] }}' + when: + - zj_quay_tag.name is regex(remove_registry_tag_regex) + # "last_modified": "Thu, 23 Mar 2023 21:59:40 -0000" + - (now() - (zj_quay_tag.last_modified | to_datetime('%a, %d %b %Y %H:%M:%S -0000'))).seconds > remove_registry_tag_age + +- name: List tags to remove + debug: + var: _to_delete + +- name: Delete tag + no_log: true + uri: + url: '{{ remove_registry_tag_api_url }}/repository/{{ _repopath }}/tag/{{ zj_quay_tag }}' + method: DELETE + status_code: [200, 204] + headers: + 'Authorization': 'Bearer {{ container_registry_credentials[_registry].api_token }}' + loop: '{{ _to_delete }}' + loop_control: + loop_var: zj_quay_tag