From 8bfffe39cf88ac401a542fca8262863b7dca55b4 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Thu, 23 Mar 2017 18:33:59 +0000 Subject: [PATCH] Add Packages file parsing lookup This is a lookup in case you need to parse a ubuntu repo Packages file for a list of packages. It's useful in osa if you want to have a mirror of an ubuntu repo, and output a list of its packages, with their location and checksum. You can use the output dict to feed another tool (for example downloading with aria). Change-Id: I10ba72f0b64267ae6afe57317e135d782445e934 --- lookup/packages_file.py | 79 +++++++++++++++++++++++++++++++++++++++++ tests/files/Packages | 44 +++++++++++++++++++++++ tests/test-lookups.yml | 40 +++++++++++++++++++++ tests/test.yml | 2 ++ 4 files changed, 165 insertions(+) create mode 100644 lookup/packages_file.py create mode 100644 tests/files/Packages create mode 100644 tests/test-lookups.yml diff --git a/lookup/packages_file.py b/lookup/packages_file.py new file mode 100644 index 0000000..85345fd --- /dev/null +++ b/lookup/packages_file.py @@ -0,0 +1,79 @@ +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2017, Jean-Philippe Evrard + +# Take a path to a debian mirror release file, and outputs a dict with +# package names, each of the packages holding the following: +# - package version +# - checksums +# - Relative location to pool folder +# +# example: +# get_url: +# url: http://rpc-repo.rackspace.com/apt-mirror/integrated/dists/r14.0.0rc1-trusty/main/binary-amd64/Packages +# dest: /tmp/trusty-amd64-Packages +# debug: +# var: item +# with_packages_file: +# - /tmp/trusty-amd64-Packages + +import os + +from ansible.plugins.lookup import LookupBase +from ansible.errors import AnsibleLookupError + +IMPORTANT_FIELDS = ['Version', 'Filename', 'MD5sum', 'SHA1', 'SHA256'] + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + +def parse_fields(line): + for field in IMPORTANT_FIELDS: + if line.startswith(field + ":"): + return (field, line.split(":")[1].strip()) + + +class LookupModule(LookupBase): + def run(self, terms, variables=None, **kwargs): + ret = [] + for term in terms: + pkg_details = {} + with open(term, 'r') as f: + for line in f: + #non empty line means pkg data + if line.strip(): + if line.startswith('Package:'): + currentpkg = line.split(":")[1].strip() + pkg_details[currentpkg] = {} + elif line.startswith('Provides:'): + pkg_details[line.split(":")[1].strip()] = pkg_details[currentpkg] + else: + # Now doing package data + parsed = parse_fields(line) + if parsed: + pkg_details[currentpkg][parsed[0]] = parsed[1] + else: + currentpkg="" + ret.append(pkg_details) + return ret + +# For debug purposes +if __name__ == '__main__': + import sys + import json + print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True)) diff --git a/tests/files/Packages b/tests/files/Packages new file mode 100644 index 0000000..b5d25c6 --- /dev/null +++ b/tests/files/Packages @@ -0,0 +1,44 @@ +Package: 0ad +Priority: optional +Section: universe/games +Installed-Size: 9487 +Maintainer: Ubuntu Developers +Original-Maintainer: Debian Games Team +Architecture: amd64 +Version: 0.0.16-2~ubuntu14.04.1 +Depends: 0ad-data (>= 0.0.16), 0ad-data (<= 0.0.16-2~ubuntu14.04.1), 0ad-data-common (>= 0.0.16), 0ad-data-common (<= 0.0.16-2~ubuntu14.04.1), libboost-filesystem1.54.0, libc6 (>= 2.15), libcurl3-gnutls (>= 7.16.2), libenet2a (>= 1.3.7+ds), libgcc1 (>= 1:4.1.1), libgl1-mesa-glx | libgl1, libgloox11, libicu52 (>= 52~m1-1~), libjpeg8 (>= 8c), libminiupnpc8 (>= 1.6), libmozjs-24-0, libnvtt2, libopenal1 (>= 1:1.13), libpng12-0 (>= 1.2.13-4), libsdl1.2debian (>= 1.2.11), libstdc++6 (>= 4.6), libvorbisfile3 (>= 1.1.2), libwxbase3.0-0 (>= 3.0.0), libwxgtk3.0-0 (>= 3.0.0), libx11-6, libxcursor1 (>> 1.1.2), libxml2 (>= 2.9.0), zlib1g (>= 1:1.2.0) +Pre-Depends: dpkg (>= 1.15.6~) +Filename: pool/main/0/0ad/0ad_0.0.16-2~ubuntu14.04.1_amd64.deb +Size: 2599794 +MD5sum: 7e5f2ba5e1a95e47753eeb962af32e26 +SHA1: 3cf898d4595092daa274e6cd8d9afd0332b0afbe +SHA256: f4602a90a305abeacb4a48bbfd7d609aa7cbb3ed2ab9127ae30ef64a4be88378 +Description: Real-time strategy game of ancient warfare +Origin: Ubuntu +Description-Md5: d943033bedada21853d2ae54a2578a7b +Homepage: http://play0ad.com/ +Bugs: https://bugs.launchpad.net/ubuntu/+filebug + +Package: aide +Priority: optional +Section: admin +Installed-Size: 1581 +Maintainer: Ubuntu Developers +Original-Maintainer: Aide Maintainers +Architecture: amd64 +Version: 0.16~a2.git20130520-2ubuntu0.1 +Provides: aide-binary +Recommends: aide-common (= 0.16~a2.git20130520-2ubuntu0.1) +Conflicts: aide-dynamic, aide-xen +Filename: pool/main/a/aide/aide_0.16~a2.git20130520-2ubuntu0.1_amd64.deb +Size: 576282 +MD5sum: 53ef6f122eb5d4da6546b7d84de40141 +SHA1: 62cfec54ee4654b3a3390430d545961a52184a9f +SHA256: 527e0d4e26edc274c18f255d2f44cd1d48b3681e37883e8d1aec2225bfc0b9ed +Description: Advanced Intrusion Detection Environment - static binary +Built-Using: acl (= 2.2.52-1), attr (= 1:2.4.47-1ubuntu1), e2fsprogs (= 1.42.9-3ubuntu1), eglibc (= 2.19-0ubuntu6.3), libselinux (= 2.2.2-1ubuntu0.1), mhash (= 0.9.9.9-4), zlib (= 1:1.2.8.dfsg-1ubuntu1) +Homepage: http://aide.sourceforge.net +Origin: Ubuntu +Supported: 5y +Description-Md5: 5cf26f0af8c95254f9ec7920c1214cbe +Bugs: https://bugs.launchpad.net/ubuntu/+filebug diff --git a/tests/test-lookups.yml b/tests/test-lookups.yml new file mode 100644 index 0000000..02de08c --- /dev/null +++ b/tests/test-lookups.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2017, Jean-Philippe Evrard + + +- name: Test lookups + hosts: localhost + connection: local + gather_facts: no + tasks: + - name: Ensure Package list contains the files listed in Package + set_fact: + packages_dict: "{{ lookup('packages_file', playbook_dir ~ '/files/Packages') }}" + - name: Validate that Filename, MD5sum, SHA1, SHA256 and Version are properly generated + assert: + that: + - packages_dict['0ad']['Filename'] == "pool/main/0/0ad/0ad_0.0.16-2~ubuntu14.04.1_amd64.deb" + - packages_dict['0ad']['MD5sum'] == "7e5f2ba5e1a95e47753eeb962af32e26" + - packages_dict['0ad']['SHA1'] == "3cf898d4595092daa274e6cd8d9afd0332b0afbe" + - packages_dict['0ad']['SHA256'] == "f4602a90a305abeacb4a48bbfd7d609aa7cbb3ed2ab9127ae30ef64a4be88378" + - packages_dict['0ad']['Version'] == "0.0.16-2~ubuntu14.04.1" + - name: Validate that a provides package is listed as a package too + assert: + that: + - packages_dict['aide'] is defined + - packages_dict['aide-binary'] is defined + - packages_dict['aide'] == packages_dict['aide-binary'] diff --git a/tests/test.yml b/tests/test.yml index 3f0389f..d01152d 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -18,4 +18,6 @@ - include: test-filters.yml +- include: test-lookups.yml + - include: test-strategy-tagfilter.yml