From 725222afee6337074c73e084c1e1c6afa0691c92 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 7 Dec 2015 13:44:10 -0600 Subject: [PATCH] first commit Signed-off-by: Kevin Carter --- CONTRIBUTING.rst | 85 ++++++ LICENSE | 202 +++++++++++++ README.rst | 17 ++ defaults/main.yml | 92 ++++++ dev-requirements.txt | 6 + doc/Makefile | 195 +++++++++++++ doc/source/conf.py | 290 +++++++++++++++++++ doc/source/index.rst | 14 + files/openstack-wheel-builder.py | 408 +++++++++++++++++++++++++++ files/openstack-wheel-builder.sh | 169 +++++++++++ files/rsync.defaults | 3 + files/rsyncd.conf | 15 + files/sshd_config | 43 +++ handlers/main.yml | 44 +++ meta/main.yml | 34 +++ readme.rst | 22 ++ run_tests.sh | 43 +++ setup.cfg | 24 ++ setup.py | 22 ++ tasks/main.yml | 31 ++ tasks/repo_install.yml | 50 ++++ tasks/repo_key_distribute.yml | 23 ++ tasks/repo_key_populate.yml | 30 ++ tasks/repo_post_install.yml | 115 ++++++++ tasks/repo_pre_install.yml | 61 ++++ tasks/repo_sync_manager.yml | 28 ++ templates/lsyncd.lua.j2 | 21 ++ templates/manual-init.override.j2 | 9 + templates/nginx.conf.j2 | 92 ++++++ templates/openstack-slushee.vhost.j2 | 25 ++ templates/rsyncd.conf.j2 | 15 + tests/ansible-role-requirements.yml | 24 ++ tests/ansible.cfg | 4 + tests/inventory | 2 + tests/test.yml | 128 +++++++++ tox.ini | 97 +++++++ 36 files changed, 2483 insertions(+) create mode 100644 CONTRIBUTING.rst create mode 100644 LICENSE create mode 100644 README.rst create mode 100644 defaults/main.yml create mode 100644 dev-requirements.txt create mode 100644 doc/Makefile create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst create mode 100755 files/openstack-wheel-builder.py create mode 100644 files/openstack-wheel-builder.sh create mode 100644 files/rsync.defaults create mode 100644 files/rsyncd.conf create mode 100644 files/sshd_config create mode 100644 handlers/main.yml create mode 100644 meta/main.yml create mode 100644 readme.rst create mode 100644 run_tests.sh create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tasks/main.yml create mode 100644 tasks/repo_install.yml create mode 100644 tasks/repo_key_distribute.yml create mode 100644 tasks/repo_key_populate.yml create mode 100644 tasks/repo_post_install.yml create mode 100644 tasks/repo_pre_install.yml create mode 100644 tasks/repo_sync_manager.yml create mode 100644 templates/lsyncd.lua.j2 create mode 100644 templates/manual-init.override.j2 create mode 100644 templates/nginx.conf.j2 create mode 100644 templates/openstack-slushee.vhost.j2 create mode 100644 templates/rsyncd.conf.j2 create mode 100644 tests/ansible-role-requirements.yml create mode 100644 tests/ansible.cfg create mode 100644 tests/inventory create mode 100644 tests/test.yml create mode 100644 tox.ini diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..e09f025 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,85 @@ +repo_server +######## +:tags: openstack, cloud, ansible +:category: \*nix + +contributor guidelines +^^^^^^^^^^^^^^^^^^^^^^ + +Filing Bugs +----------- + +Bugs should be filed on Launchpad, not GitHub: "https://bugs.launchpad.net/openstack-ansible" + + +When submitting a bug, or working on a bug, please ensure the following criteria are met: + * The description clearly states or describes the original problem or root cause of the problem. + * Include historical information on how the problem was identified. + * Any relevant logs are included. + * The provided information should be totally self-contained. External access to web services/sites should not be needed. + * Steps to reproduce the problem if possible. + + +Submitting Code +--------------- + +Changes to the project should be submitted for review via the Gerrit tool, following +the workflow documented at: "http://docs.openstack.org/infra/manual/developers.html#development-workflow" + +Pull requests submitted through GitHub will be ignored and closed without regard. + + +Extra +----- + +Tags: + If it's a bug that needs fixing in a branch in addition to Master, add a '\-backport-potential' tag (eg ``juno-backport-potential``). There are predefined tags that will autocomplete. + +Status: + Please leave this alone, it should be New till someone triages the issue. + +Importance: + Should only be touched if it is a Blocker/Gating issue. If it is, please set to High, and only use Critical if you have found a bug that can take down whole infrastructures. + + +Style guide +----------- + +When creating tasks and other roles for use in Ansible please create then using the YAML dictionary format. + +Example YAML dictionary format: + .. code-block:: yaml + + - name: The name of the tasks + module_name: + thing1: "some-stuff" + thing2: "some-other-stuff" + tags: + - some-tag + - some-other-tag + + +Example **NOT** in YAML dictionary format: + .. code-block:: yaml + + - name: The name of the tasks + module_name: thing1="some-stuff" thing2="some-other-stuff" + tags: + - some-tag + - some-other-tag + + +Usage of the ">" and "|" operators should be limited to Ansible conditionals and command modules such as the ansible ``shell`` module. + + +Issues +------ + +When submitting an issue, or working on an issue please ensure the following criteria are met: + * The description clearly states or describes the original problem or root cause of the problem. + * Include historical information on how the problem was identified. + * Any relevant logs are included. + * If the issue is a bug that needs fixing in a branch other than Master, add the ‘backport potential’ tag TO THE ISSUE (not the PR). + * The provided information should be totally self-contained. External access to web services/sites should not be needed. + * If the issue is needed for a hotfix release, add the 'expedite' label. + * Steps to reproduce the problem if possible. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8f71f43 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f07c3e6 --- /dev/null +++ b/README.rst @@ -0,0 +1,17 @@ +OpenStack repo server +##################### +:tags: openstack, repo, server, cloud, ansible +:category: \*nix + +Role to deploy a repository server for both python packages and git sources. + +.. code-block:: yaml + + - name: Setup repo servers + hosts: repo_all + user: root + roles: + - { role: "repo_server", tags: [ "repo-server" ] } + vars: + memcached_servers: 127.0.0.1:11211 + memcached_encryption_key: secrete diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..573fb7d --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,92 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# If ``repo_nginx_threads`` is unset the system will use half the number of available VCPUS to +# compute the number of api workers to use. +# repo_nginx_threads: 16 + +# Defines that the role will be deployed on a host machine +is_metal: true + +repo_worker_connections: 1024 +repo_server_name: openstack-slushee + +repo_service_home_folder: /var/www +repo_service_user_name: nginx +repo_service_group_name: www-data + +repo_auto_rebuild: false + +repo_memcached_servers: "{% for host in groups['repo_all'] %}{{ hostvars[host]['ansible_ssh_host'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" + +# If you want to regenerate the repo users SSH keys, on each run, set this var to True +# Otherwise keys will be generated on the first run and not regenerated each run. +repo_recreate_keys: False + +repo_apt_packages: + - aptitude + - bridge-utils + - build-essential + - cgroup-lite + - cmake + - dmeventd + - fcgiwrap + - git-core + - iptables + - lsyncd + - lynx + - nginx-extras + - rsync + - sshpass + - tmux + - liberasurecode1 + - liberasurecode-dev + - libldap2-dev + - libsasl2-dev + - libssl-dev + - libxslt1.1 + - libpq-dev + - libffi-dev + - libsqlite3-dev + - libxml2-dev + - libxslt1-dev + - libkmod-dev + - libkrb5-dev + - libkmod2 + - libvirt-dev + - libmariadbclient-dev + - lxc-dev + - lvm2 + - python-dev + - python-software-properties + - sqlite3 + - swig + - vlan + - uuid-dev + +repo_pip_packages: + - cloudlib + - PyCrypto + - python-memcached + - PyYAML + - requests + - turbolift + - wheel + - yaprt + - virtualenv + - virtualenv-tools + +# Main web server port +repo_server_port: 8181 diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..f9f762e --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,6 @@ +ansible-lint +ansible>=1.9.1,<2.0.0 + +# this is required for the docs build jobs +sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 +oslosphinx>=2.5.0 # Apache-2.0 diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..dd215ca --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,195 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/openstack-ansible-repo_server.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/openstack-ansible-repo_server.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/openstack-ansible-repo_server" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/openstack-ansible-repo_server" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +livehtml: html + sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..d38823d --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# openstack-ansible-repo_server documentation build configuration file, created by +# sphinx-quickstart on Mon Apr 13 20:42:26 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'oslosphinx' +] + +# The link to the browsable source code (for the left hand menu) +oslosphinx_cgit_link = 'http://git.openstack.org/cgit/openstack/openstack-ansible-repo_server' + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'openstack-ansible-repo_server' +copyright = '2015, openstack-ansible-repo_server contributors' +author = 'openstack-ansible-repo_server contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = 'master' +# The full version, including alpha/beta/rc tags. +release = 'master' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'openstack-ansible-repo_serverdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', + + # Latex figure (float) alignment + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'openstack-ansible-repo_server.tex', + 'openstack-ansible-repo_server Documentation', + 'openstack-ansible-repo_server contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'openstack-ansible-repo_server', + 'openstack-ansible-repo_server Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'openstack-ansible-repo_server', + 'openstack-ansible-repo_server Documentation', + author, 'openstack-ansible-repo_server', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..1236e62 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,14 @@ +repo_server Docs +============= + +Tell us what the role is supposed to do? + +Basic Role Example +^^^^^^^^^^^^^^^^^^ + +Tell us how to use the role. + +.. code-block:: yaml + + - role: "$ROLENAME" + ROLE_VARS... diff --git a/files/openstack-wheel-builder.py b/files/openstack-wheel-builder.py new file mode 100755 index 0000000..f73cf66 --- /dev/null +++ b/files/openstack-wheel-builder.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2015, Kevin Carter + +import os +import traceback + +import yaml + +from cloudlib import arguments +from cloudlib import shell + + +REQUIREMENTS_FILE_TYPES = [ + 'requirements.txt', + 'global-requirements.txt', + 'test-requirements.txt', + 'dev-requirements.txt' +] + + +# List of variable names that could be used within the yaml files that +# represent lists of python packages. +BUILT_IN_PIP_PACKAGE_VARS = [ + 'service_pip_dependencies', + 'pip_common_packages', + 'pip_container_packages', + 'pip_packages' +] + + +class DependencyFileProcessor(object): + def __init__(self, local_path): + """Find and process dependent files from a local_path. + + :type local_path: ``str`` + :return: + """ + self.pip = dict() + self.pip['git_package'] = list() + self.pip['py_package'] = list() + self.git_pip_install = 'git+%s@%s' + self.file_names = self._get_files(path=local_path) + + # Process everything simply by calling the method + self._process_files(ext=('yaml', 'yml')) + + def _filter_files(self, file_names, ext): + """Filter the files and return a sorted list. + + :type file_names: + :type ext: ``str`` or ``tuple`` + :returns: ``list`` + """ + _file_names = list() + for file_name in file_names: + if file_name.endswith(ext): + if '/defaults/' in file_name or '/vars/' in file_name: + _file_names.append(file_name) + else: + continue + elif os.path.basename(file_name) in REQUIREMENTS_FILE_TYPES: + with open(file_name, 'rb') as f: + packages = [ + i.split()[0] for i in f.read().splitlines() + if i + if not i.startswith('#') + ] + self.pip['py_package'].extend(packages) + else: + return sorted(_file_names, reverse=True) + + @staticmethod + def _get_files(path): + """Return a list of all files in the defaults/repo_packages directory. + + :type path: ``str`` + :returns: ``list`` + """ + paths = os.walk(os.path.abspath(path)) + files = list() + for fpath, _, afiles in paths: + for afile in afiles: + files.append(os.path.join(fpath, afile)) + else: + return files + + def _check_plugins(self, git_repo_plugins, git_data): + """Check if the git url is a plugin type. + + :type git_repo_plugins: ``dict`` + :type git_data: ``dict`` + """ + for repo_plugin in git_repo_plugins: + plugin = '%s/%s' % ( + repo_plugin['path'].strip('/'), + repo_plugin['package'].lstrip('/') + ) + + package = self.git_pip_install % ( + git_data['repo'], + '%s#egg=%s&subdirectory=%s' % ( + git_data['branch'], + repo_plugin['package'].strip('/'), + plugin + ) + ) + + self.pip['git_package'].append(package) + + def _process_git(self, loaded_yaml, git_item): + """Process git repos. + + :type loaded_yaml: ``dict`` + :type git_item: ``str`` + """ + git_data = dict() + if git_item.split('_')[0] == 'git': + var_name = 'git' + else: + var_name = git_item.split('_')[0] + + git_data['repo'] = loaded_yaml.get(git_item) + git_data['branch'] = loaded_yaml.get( + '%s_git_install_branch' % var_name.replace('.', '_') + ) + if not git_data['branch']: + git_data['branch'] = loaded_yaml.get( + 'git_install_branch', + 'master' + ) + + package = self.git_pip_install % ( + git_data['repo'], git_data['branch'] + ) + + self.pip['git_package'].append(package) + + git_repo_plugins = loaded_yaml.get('%s_repo_plugins' % var_name) + if git_repo_plugins: + self._check_plugins( + git_repo_plugins=git_repo_plugins, + git_data=git_data + ) + + def _process_files(self, ext): + """Process files. + + :type ext: ``tuple`` + """ + file_names = self._filter_files( + file_names=self.file_names, + ext=ext + ) + + for file_name in file_names: + with open(file_name, 'rb') as f: + loaded_config = yaml.safe_load(f.read()) + + for key, values in loaded_config.items(): + if key.endswith('git_repo'): + self._process_git( + loaded_yaml=loaded_config, + git_item=key + ) + + if [i for i in BUILT_IN_PIP_PACKAGE_VARS if i in key]: + self.pip['py_package'].extend(values) + + +def _arguments(): + """Return CLI arguments.""" + + arguments_dict = { + 'optional_args': { + 'local_path': { + 'commands': [ + '--local-path' + ], + 'help': 'Local path to cloned code.', + 'metavar': '[PATH]', + 'required': True + }, + 'report_file': { + 'commands': [ + '--report-file' + ], + 'help': 'Full path to write the package report to', + 'metavar': '[FILE_PATH]', + 'required': True + }, + 'storage_pool': { + 'commands': [ + '--storage-pool' + ], + 'help': 'Full path to the directory where you want to store' + ' built wheels.', + 'metavar': '[PATH]', + 'required': True + }, + 'release_directory': { + 'commands': [ + '--release-directory' + ], + 'help': 'Full path to the directory where the releaesed links' + ' will be stored.', + 'metavar': '[PATH]', + 'required': True + }, + 'add_on_repos': { + 'commands': [ + '--add-on-repos' + ], + 'help': 'Full repo path to require as an additional add on' + ' repo. Example:' + ' "git+https://github.com/rcbops/other-repo@master"', + 'metavar': '[REPO_NAME]', + 'nargs': '+' + }, + 'link_pool': { + 'commands': [ + '--link-pool' + ], + 'help': 'Full path to the directory links are stored.', + 'metavar': '[PATH]', + 'required': True + } + } + } + + return arguments.ArgumentParserator( + arguments_dict=arguments_dict, + epilog='Licensed Apache2', + title='Discover all of the requirements within the' + ' openstack-ansible project.', + detail='Requirement lookup', + description='Discover all of the requirements within the' + ' openstack-ansible project.', + env_name='OS_ANSIBLE' + ).arg_parser() + + +def _abs_path(path): + return os.path.abspath( + os.path.expanduser( + path + ) + ) + + +def _run_command(command): + print('Running "%s"' % command[2]) + run_command = shell.ShellCommands(debug=True) + info, success = run_command.run_command(' '.join(command)) + if not success: + raise SystemExit(info) + else: + print(info) + + +def main(): + """Run the main application.""" + user_vars = _arguments() + return_list = list() + try: + dfp = DependencyFileProcessor( + local_path=_abs_path(user_vars['local_path']) + ) + return_list.extend(dfp.pip['py_package']) + return_list.extend(dfp.pip['git_package']) + except Exception as exp: + raise SystemExit( + 'Execution failure. Path: "%s", Error: "%s", Trace:\n%s' % ( + user_vars['local_path'], + str(exp), + traceback.format_exc() + ) + ) + else: + return_data = { + 'packages': list(), + 'remote_packages': list() + } + for file_name in sorted(set(return_list)): + is_url = file_name.startswith(('http:', 'https:', 'git+')) + if is_url: + if '@' not in file_name: + return_data['packages'].append(file_name) + else: + return_data['remote_packages'].append(file_name) + else: + return_data['packages'].append(file_name) + else: + return_data['packages'] = ' '.join( + ['"%s"' % i for i in set(return_data['packages'])] + ) + + if user_vars['add_on_repos']: + return_data['remote_packages'].extend( + [i.strip() for i in user_vars['add_on_repos']] + ) + + return_data['remote_packages'] = ' '.join( + ['"%s"' % i for i in set(return_data['remote_packages'])] + ) + + # Build report + report_command = [ + 'yaprt', + '--debug', + 'create-report', + '--report-file', + _abs_path(user_vars['report_file']), + '--git-install-repos', + return_data['remote_packages'], + '--packages', + return_data['packages'] + ] + _run_command(report_command) + + # Build requirements wheels + requirements_command = [ + 'yaprt', + '--debug', + 'build-wheels', + '--report-file', + _abs_path(user_vars['report_file']), + '--storage-pool', + _abs_path(user_vars['storage_pool']), + '--link-dir', + _abs_path(user_vars['release_directory']), + '--pip-extra-link-dirs', + _abs_path(user_vars['link_pool']), + '--pip-index', + 'https://rpc-repo.rackspace.com/pools', + '--pip-extra-index', + 'https://pypi.python.org/simple', + '--pip-bulk-operation', + '--build-output', + '/tmp/openstack-wheel-output', + '--build-dir', + '/tmp/openstack-builder', + '--build-requirements', + '--force-clean' + ] + _run_command(requirements_command) + + # Build wheels from git-repos + requirements_command = [ + 'yaprt', + '--debug', + 'build-wheels', + '--report-file', + _abs_path(user_vars['report_file']), + '--storage-pool', + _abs_path(user_vars['storage_pool']), + '--link-dir', + _abs_path(user_vars['release_directory']), + '--pip-extra-link-dirs', + _abs_path(user_vars['link_pool']), + '--pip-no-deps', + '--pip-no-index', + '--build-output', + '/tmp/openstack-wheel-output', + '--build-dir', + '/tmp/openstack-builder', + '--build-branches', + '--build-releases', + '--force-clean' + ] + _run_command(requirements_command) + + # Create HTML index for all files in the release directory + index_command = [ + 'yaprt', + '--debug', + 'create-html-indexes', + '--repo-dir', + _abs_path(user_vars['release_directory']) + ] + _run_command(index_command) + + # Store the git repositories + index_command = [ + 'yaprt', + 'store-repos', + '--report-file', + _abs_path(user_vars['report_file']), + '--git-repo-path', + '/var/www/repo/openstackgit' + ] + _run_command(index_command) + +if __name__ == '__main__': + main() diff --git a/files/openstack-wheel-builder.sh b/files/openstack-wheel-builder.sh new file mode 100644 index 0000000..b4abf2a --- /dev/null +++ b/files/openstack-wheel-builder.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Notes: +# To use this script you MUST move it to some path that will be called. +# I recommend that the script be stored and executed from +# "/opt/openstack-wheel-builder.sh". This script is a wrapper script that relies +# on the "openstack-wheel-builder.py" and is execute from +# "/opt/openstack-wheel-builder.py". + +# Overrides: +# This script has several things that can be overriden via environment +# variables. +# Git repository that the rcbops ansible lxc source code will be cloned from. +# This repo should be a repo that is available via HTTP. +# GIT_REPO="" + +# The URI for the github api. This is ONLY used when the $RELEASES variable +# is an empty string. Which causes the script to go discover the available +# releases. +# GITHUB_API_ENDPOINT="" + +# Local directory to store the source code while interacting with it. +# WORK_DIR="" + +# Local directory to store the built wheels. +# OUTPUT_WHEEL_PATH="" + +# Space seperated list of all releases to build for. If unset the releases +# will be discovered. +# RELEASES="" + +# Space seperated list of all releases to exclude from building. This is +# ONLY used when the $RELEASES variable is an empty string. +# EXCLUDE_RELEASES="" + +set -e -o -v + +# Trap any errors that might happen in executing the script +trap my_trap_handler ERR + +# Ensure there is a base path loaded +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +# Defined variables. +GIT_REPO="${GIT_REPO:-https://github.com/openstack/openstack-ansible}" +GITHUB_API_ENDPOINT="${GITHUB_API_ENDPOINT:-https://api.github.com/repos/openstack/openstack-ansible}" + +# Predefined working directory. +WORK_DIR="${WORK_DIR:-/tmp/openstack-ansible-deployment}" + +# Output directories. +OUTPUT_WHEEL_PATH="${OUTPUT_WHEEL_PATH:-/var/www/repo/os-releases}" +LINK_PATH="${LINK_PATH:-/var/www/repo/links}" +REPORT_DIR="${REPORT_DIR:-/var/www/repo/reports}" +STORAGE_POOL="${STORAGE_POOL:-/var/www/repo/pools}" + +# Additional space separated list of repos to always include in a build. +ADDON_REPOS="git+https://github.com/rcbops/horizon-extensions.git@master " + +# Set the force build option to false +FORCE_BUILD="${FORCE_BUILD:-false}" + +# Default is an empty string which causes the script to go discover the available +# branches from the github API. +RELEASES=${RELEASES:-""} + +# Define branches that you no longer want new wheels built for or checked against. +EXCLUDE_RELEASES="${EXCLUDE_RELEASES:-v9.0.0 gh-pages revert}" + +# Name of the lock file. +LOCKFILE="/tmp/wheel_builder.lock" + +function my_trap_handler { + kill_job +} + +function lock_file_remove { + if [ -f "${LOCKFILE}" ]; then + rm "${LOCKFILE}" + fi +} + +function kill_job { + set +e + # If the job needs killing kill the pid and unlock the file. + if [ -f "${LOCKFILE}" ]; then + PID=$(cat ${LOCKFILE}) + lock_file_remove + kill -9 "${PID}" + fi +} + +function cleanup { + # Ensure workspaces are cleaned up + rm -rf /tmp/openstack_wheels* + rm -rf /tmp/pip* + rm -rf "${WORK_DIR}" +} + +# Check for releases +if [ -z "${RELEASES}" ];then + echo "No releases specified. Provide a space separated list branches to build for." + exit 1 +fi + +# Check for system lock file. +if [ ! -f "${LOCKFILE}" ]; then + echo $$ | tee "${LOCKFILE}" +else + if [ "$(find ${LOCKFILE} -mmin +240)" ]; then + logger "Stale pid found for ${LOCKFILE}." + logger "Killing any left over processes and unlocking" + kill_job + else + NOTICE="Active job already in progress. Check pid \"$(cat ${LOCKFILE})\" for status. Lock file: ${LOCKFILE}" + echo $NOTICE + logger ${NOTICE} + exit 1 + fi +fi + +# Iterate through the list of releases and build everything that's needed +logger "Building Python Wheels for ${RELEASES}" +for release in ${RELEASES}; do + + if [ ! -d "${OUTPUT_WHEEL_PATH}/${release}" ] || [[ "${FORCE_BUILD}" == "true" ]]; then + # Perform cleanup + cleanup + + # Git clone repo + git clone "${GIT_REPO}" "${WORK_DIR}" + + # checkout release + pushd "${WORK_DIR}" + git checkout "${release}" + popd + + # Build wheels + OVERRIDE_WHEEL_OUTPUT_PATH="${OVERRIDE_WHEEL_OUTPUT_PATH:-${OUTPUT_WHEEL_PATH}/${release}}" + mkdir -p "${OVERRIDE_WHEEL_OUTPUT_PATH}" + /opt/openstack-wheel-builder.py --report-file "${REPORT_DIR}/${release}.json" \ + --link-pool "${LINK_PATH}" \ + --local-path "${WORK_DIR}" \ + --storage-pool ${STORAGE_POOL} \ + --release-directory "${OVERRIDE_WHEEL_OUTPUT_PATH}" \ + --add-on-repos ${ADDON_REPOS} + + fi + echo "Complete [ ${release} ]" +done + +# Perform cleanup +cleanup + +# Remove lock file on job completion +lock_file_remove diff --git a/files/rsync.defaults b/files/rsync.defaults new file mode 100644 index 0000000..901038a --- /dev/null +++ b/files/rsync.defaults @@ -0,0 +1,3 @@ +RSYNC_ENABLE=true +RSYNC_CONFIG_FILE=/etc/rsyncd.conf +RSYNC_OPTS='' diff --git a/files/rsyncd.conf b/files/rsyncd.conf new file mode 100644 index 0000000..12a2815 --- /dev/null +++ b/files/rsyncd.conf @@ -0,0 +1,15 @@ +log file = /var/log/rsyncd.log +pid file = /var/run/rsyncd.pid +lock file = /var/run/rsync.lock + +max connections = 15 +timeout = 600 +transfer logging = true + +[openstack_mirror] +path = {{ repo_service_home_folder }}/repo +comment = OpenStack Private Cloud Mirror Server +use chroot = yes +read only = yes +uid = nobody +gid = nogroup diff --git a/files/sshd_config b/files/sshd_config new file mode 100644 index 0000000..78d32b5 --- /dev/null +++ b/files/sshd_config @@ -0,0 +1,43 @@ +Port 22 +Protocol 2 +HostKey /etc/ssh/ssh_host_rsa_key +HostKey /etc/ssh/ssh_host_dsa_key +HostKey /etc/ssh/ssh_host_ecdsa_key +HostKey /etc/ssh/ssh_host_ed25519_key +UsePrivilegeSeparation yes + +KeyRegenerationInterval 3600 +ServerKeyBits 1024 + +SyslogFacility AUTH +LogLevel INFO + +LoginGraceTime 120 +PermitRootLogin yes +StrictModes yes + +RSAAuthentication yes +PubkeyAuthentication yes + +IgnoreRhosts yes +RhostsRSAAuthentication no +HostbasedAuthentication no + +PermitEmptyPasswords no + +ChallengeResponseAuthentication no + +PasswordAuthentication no + +X11Forwarding yes +X11DisplayOffset 10 +PrintMotd no +PrintLastLog yes +TCPKeepAlive yes + +AcceptEnv LANG LC_* + +Subsystem sftp /usr/lib/openssh/sftp-server + +UsePAM yes +UseDNS no diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..5aeca10 --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1,44 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: reload nginx + service: + name: "nginx" + state: restarted + pattern: "nginx" + +- name: reload rsyncd + service: + name: "rsync" + state: restarted + pattern: "rsync" + +- name: reload fcgiwrap + service: + name: "fcgiwrap" + state: restarted + pattern: "fcgiwrap" + +- name: reload ssh + service: + name: "ssh" + state: restarted + pattern: "ssh" + +- name: reload lsyncd + service: + name: "lsyncd" + state: restarted + pattern: "lsyncd" diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..ba9696f --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,34 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: rcbops + description: Install package repo server + company: Rackspace + license: Apache2 + min_ansible_version: 1.6.6 + platforms: + - name: Ubuntu + versions: + - trusty + categories: + - cloud + - python + - development + - openstack +dependencies: + - apt_package_pinning + - pip_install + - memcached_server diff --git a/readme.rst b/readme.rst new file mode 100644 index 0000000..49049df --- /dev/null +++ b/readme.rst @@ -0,0 +1,22 @@ +repo_server role +############# +:tags: openstack, cloud, ansible, repo_server +:category: \*nix + +repo_server Role + +.. code-block:: yaml + + - name: repo_server role + hosts: "hosts" + user: root + roles: + - { role: "repo_server" } + + +Note. The template role has the template name within it. Please change the name +throughout the code base. + +.. code-block:: bsah + + find . -type f -exec sed -i 's/repo_server/CHANGE_ME_PLEASE/g' {} \; diff --git a/run_tests.sh b/run_tests.sh new file mode 100644 index 0000000..f0a8f3e --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euov + +FUNCTIONAL_TEST=${FUNCTIONAL_TEST:-true} + +# prep the host +if [ "$(which apt-get)" ]; then + apt-get install -y build-essential python2.7 python-dev git-core +fi + +# get pip, if necessary +if [ ! "$(which pip)" ]; then + curl --silent --show-error --retry 5 \ + https://bootstrap.pypa.io/get-pip.py | sudo python2.7 +fi + +# install tox +pip install tox + +# run through each tox env and execute the test +for tox_env in $(awk -F= '/envlist/ {print $2}' tox.ini | sed 's/,/ /g'); do + if [ "${tox_env}" != "ansible-functional" ]; then + tox -e ${tox_env} + elif [ "${tox_env}" == "ansible-functional" ]; then + if ${FUNCTIONAL_TEST}; then + tox -e ${tox_env} + fi + fi +done diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b746bbd --- /dev/null +++ b/setup.cfg @@ -0,0 +1,24 @@ +[metadata] +name = openstack-ansible-repo_server +summary = repo_server for OpenStack Ansible +description-file = + README.rst +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Intended Audience :: Developers + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + +[build_sphinx] +all_files = 1 +build-dir = doc/build +source-dir = doc/source + +[pbr] +warnerrors = True + +[wheel] +universal = 1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..70c2b3f --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..8b43342 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: repo_pre_install.yml +- include: repo_install.yml +- include: repo_post_install.yml + +- include: repo_key_populate.yml + +- include: repo_key_distribute.yml + when: > + groups.repo_all|length > 1 + +- include: repo_sync_manager.yml + when: > + inventory_hostname == groups['repo_all'][0] + +- name: Flush handlers + meta: flush_handlers diff --git a/tasks/repo_install.yml b/tasks/repo_install.yml new file mode 100644 index 0000000..33dd72b --- /dev/null +++ b/tasks/repo_install.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Update apt sources + apt: + update_cache: yes + cache_valid_time: 600 + register: apt_update + until: apt_update|success + retries: 5 + delay: 2 + tags: + - repo-apt-packages + +- name: Install apt packages + apt: + pkg: "{{ item }}" + state: latest + register: install_packages + until: install_packages|success + retries: 5 + delay: 5 + with_items: repo_apt_packages + tags: + - repo-apt-packages + +- name: Install pip packages + pip: + name: "{{ item }}" + state: present + extra_args: "{{ pip_install_options|default('') }}" + register: install_packages + until: install_packages|success + retries: 5 + delay: 5 + with_items: repo_pip_packages + tags: + - repo-pip-packages diff --git a/tasks/repo_key_distribute.yml b/tasks/repo_key_distribute.yml new file mode 100644 index 0000000..a8f371f --- /dev/null +++ b/tasks/repo_key_distribute.yml @@ -0,0 +1,23 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create authorized keys file from host vars + authorized_key: + user: "{{ repo_service_user_name }}" + key: "{{ hostvars[item]['repo_pubkey'] }}" + with_items: groups['repo_all'] + tags: + - repo-key + - repo-key-store diff --git a/tasks/repo_key_populate.yml b/tasks/repo_key_populate.yml new file mode 100644 index 0000000..2036e3f --- /dev/null +++ b/tasks/repo_key_populate.yml @@ -0,0 +1,30 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get public key contents and store as var + command: | + cat {{ repo_service_home_folder }}/.ssh/id_rsa.pub + register: repo_pub + changed_when: false + tags: + - repo-key + - repo-key-create + +- name: Register a fact for the repo user pub key + set_fact: + repo_pubkey: "{{ repo_pub.stdout }}" + tags: + - repo-key + - repo-key-create diff --git a/tasks/repo_post_install.yml b/tasks/repo_post_install.yml new file mode 100644 index 0000000..327e87f --- /dev/null +++ b/tasks/repo_post_install.yml @@ -0,0 +1,115 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: create the system group + group: + name: "{{ repo_service_group_name }}" + state: "present" + system: "yes" + tags: + - pkg-repo-group + +- name: Remove old key file(s) if found + file: + path: "{{ item }}" + state: "absent" + with_items: + - "{{ repo_service_home_folder }}/.ssh/authorized_keys" + - "{{ repo_service_home_folder }}/.ssh/id_rsa" + - "{{ repo_service_home_folder }}/.ssh/id_rsa.pub" + when: repo_recreate_keys | bool + tags: + - repo-key + - repo-key-create + +- name: Create the nginx system user + user: + name: "{{ repo_service_user_name }}" + group: "{{ repo_service_group_name }}" + comment: "Nginx repo user" + shell: "/bin/bash" + system: "yes" + createhome: "yes" + home: "{{ repo_service_home_folder }}" + generate_ssh_key: "yes" + tags: + - pkg-repo-user + - repo-key + - repo-key-create + +- name: File and directory setup + file: + path: "{{ item.path }}" + state: "{{ item.state }}" + owner: "{{ item.owner|default('root') }}" + group: "{{ item.group|default('root') }}" + mode: "{{ item.mode }}" + with_items: + - { path: "{{ repo_service_home_folder }}", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/.ssh", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2700" } + - { path: "{{ repo_service_home_folder }}/repo", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/links", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/os-releases", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/os-releases/{{ openstack_release }}", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/openstackgit", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/python_packages", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/python_packages/pools", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/pools", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "{{ repo_service_home_folder }}/repo/reports", state: "directory", owner: "{{ repo_service_user_name }}", group: "{{ repo_service_group_name }}", mode: "2755" } + - { path: "/etc/lsyncd", state: "directory", mode: "0755" } + - { path: "/var/log/lsyncd", state: "directory", mode: "0755" } + - { path: "/etc/nginx/sites-enabled/default", state: "absent", mode: "0644" } + tags: + - pkg-repo-dirs + +- name: Drop NGINX configuration files + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - { src: "nginx.conf.j2", dest: "/etc/nginx/nginx.conf" } + - { src: "rsyncd.conf.j2", dest: "/etc/rsyncd.conf" } + - { src: "openstack-slushee.vhost.j2", dest: "/etc/nginx/sites-available/openstack-slushee.vhost" } + notify: + - reload nginx + tags: + - pkg-repo-nginx + - pkg-repo-config + +- name: Change fcgiwrap GID/UID + lineinfile: + dest: "/etc/init.d/fcgiwrap" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - { regexp: "^FCGI_SOCKET_OWNER=", line: 'FCGI_SOCKET_OWNER="{{ repo_service_user_name }}"' } + - { regexp: "^FCGI_SOCKET_GROUP=", line: 'FCGI_SOCKET_GROUP="{{ repo_service_group_name }}"' } + notify: + - reload nginx + - reload fcgiwrap + tags: + - pkg-repo-nginx + - pkg-repo-config + +- name: Enable openstack-slushee site + file: + src: "/etc/nginx/sites-available/openstack-slushee.vhost" + dest: "/etc/nginx/sites-enabled/openstack-slushee.vhost" + state: "link" + notify: + - reload nginx + tags: + - pkg-repo-nginx + - pkg-repo-config diff --git a/tasks/repo_pre_install.yml b/tasks/repo_pre_install.yml new file mode 100644 index 0000000..e31a803 --- /dev/null +++ b/tasks/repo_pre_install.yml @@ -0,0 +1,61 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Ensure ssh config + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - { src: "sshd_config", dest: "/etc/ssh/sshd_config" } + notify: + - reload ssh + tags: + - repo-ssh + +- name: Drop rsyncd configuration file(s) + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - { src: "rsync.defaults", dest: "/etc/default/rsync" } + notify: + - reload rsyncd + tags: + - repo-config + +- name: Copy wheel builder into place + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "root" + group: "root" + mode: "0755" + with_items: + - { src: "openstack-wheel-builder.sh", dest: "/opt/openstack-wheel-builder.sh" } + - { src: "openstack-wheel-builder.py", dest: "/opt/openstack-wheel-builder.py" } + tags: + - repo-wheel-builder + +- name: Drop base config file(s) + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ item.owner|default('root') }}" + group: "{{ item.group|default('root') }}" + mode: "{{ item.mode|default('0644') }}" + with_items: + - { src: manual-init.override.j2, dest: "/etc/init/lxc-net.override" } + tags: + - repo-config diff --git a/tasks/repo_sync_manager.yml b/tasks/repo_sync_manager.yml new file mode 100644 index 0000000..821669b --- /dev/null +++ b/tasks/repo_sync_manager.yml @@ -0,0 +1,28 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Drop lsyncd configuration file + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - { src: "lsyncd.lua.j2", dest: "/etc/lsyncd/lsyncd.conf.lua" } + notify: + - reload lsyncd + when: > + groups['repo_all']|length > 1 + tags: + - repo-lsyncd + - repo-config diff --git a/templates/lsyncd.lua.j2 b/templates/lsyncd.lua.j2 new file mode 100644 index 0000000..9202de0 --- /dev/null +++ b/templates/lsyncd.lua.j2 @@ -0,0 +1,21 @@ +settings { + logfile = "/var/log/lsyncd/lsyncd.log", + statusFile = "/var/log/lsyncd/lsyncd-status.log", + statusInterval = 20 +} + +{% for node in groups['repo_all'] %} +{% if groups['repo_all'][0] != node %} +sync { + default.rsync, + source = "{{ repo_service_home_folder }}/repo", + target = "{{ hostvars[node]['ansible_ssh_host'] }}:{{ repo_service_home_folder }}/repo", + rsync = { + compress = true, + acls = true, + rsh = "/usr/bin/ssh -l {{ repo_service_user_name }} -i {{ repo_service_home_folder }}/.ssh/id_rsa -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=5" + } +} + +{% endif %} +{% endfor %} diff --git a/templates/manual-init.override.j2 b/templates/manual-init.override.j2 new file mode 100644 index 0000000..306c937 --- /dev/null +++ b/templates/manual-init.override.j2 @@ -0,0 +1,9 @@ +# {{ ansible_managed }} + +pre-start script + echo "pass" +end script + +post-stop script + echo "pass" +end script diff --git a/templates/nginx.conf.j2 b/templates/nginx.conf.j2 new file mode 100644 index 0000000..77d7e26 --- /dev/null +++ b/templates/nginx.conf.j2 @@ -0,0 +1,92 @@ +user {{ repo_service_user_name }} {{ repo_service_group_name }}; +pid /var/run/nginx.pid; + +{% set _api_threads = ansible_processor_vcpus|default(2) // 2 %} +{% set api_threads = _api_threads if _api_threads > 0 else 1 %} + +worker_processes {{ repo_nginx_threads | default(api_threads) }}; + +events { + worker_connections {{ repo_worker_connections }}; + multi_accept on; +} + +http { + # Basic Settings + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile off; + tcp_nopush on; + tcp_nodelay on; + client_header_timeout 10m; + client_body_timeout 10m; + send_timeout 10m; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + proxy_connect_timeout 600s; + fastcgi_send_timeout 600s; + fastcgi_read_timeout 600s; + + fastcgi_cache_path /var/run/nginx-cache levels=1:2 keys_zone=OSGITREPO:100m inactive=60m; + fastcgi_cache_key "$scheme$request_method$host$request_uri"; + fastcgi_cache_use_stale error timeout invalid_header http_500; + fastcgi_ignore_headers Cache-Control Expires Set-Cookie; + + # Added Larger Post Max + client_max_body_size 5M; + + keepalive_timeout 60 15; + types_hash_max_size 2048; + + # Logging Settings + log_format gzip '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $bytes_sent ' + '"$http_referer" "$http_user_agent" "$gzip_ratio"'; + access_log /var/log/{{ repo_service_user_name }}/access.log gzip buffer=32k; + error_log /var/log/{{ repo_service_user_name }}/error.log notice; + + # Gzip Settings + gzip on; + gzip_disable "msie6"; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_min_length 1100; + gzip_buffers 16 8k; + + # Turn on gzip for all content types that should benefit from it. + gzip_types application/ecmascript; + gzip_types application/javascript; + gzip_types application/json; + gzip_types application/pdf; + gzip_types application/postscript; + gzip_types application/x-javascript; + gzip_types image/svg+xml; + gzip_types text/css; + gzip_types text/csv; + + # "gzip_types text/html" is assumed. + gzip_types text/javascript; + gzip_types text/plain; + gzip_types text/xml; + + gzip_http_version 1.0; + + # Path to NGINX Cache + fastcgi_cache_path /var/cache/nginx/ levels=1:2 keys_zone=NGINX_CACHE:10m max_size=32m inactive=5m; + + # Custom Header + add_header X-WHOAMI "OpenStack Private Slushee Repository {{ inventory_hostname }}"; + + # Virtual Host Configs + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; + + # Map Mobile + map $http_user_agent $is_mobile { + default 0; + ~*android|ip(hone|od)|windows\s+(?:ce|phone) 1; + ~*symbian|sonyericsson|samsung|lg|blackberry 1; + ~*mobile 1; + } +} diff --git a/templates/openstack-slushee.vhost.j2 b/templates/openstack-slushee.vhost.j2 new file mode 100644 index 0000000..576a4de --- /dev/null +++ b/templates/openstack-slushee.vhost.j2 @@ -0,0 +1,25 @@ +server { + listen {{ repo_server_port }}; + server_name {{ repo_server_name }}; + + # Logging + access_log /var/log/{{ repo_service_user_name }}/{{ repo_server_name }}.access.log gzip buffer=32k; + error_log /var/log/{{ repo_service_user_name }}/{{ repo_server_name }}.error.log notice; + + location / { + root {{ repo_service_home_folder }}/repo/; + autoindex on; + expires 5h; + } + + location ~ /openstackgit(/.*) { + gzip off; + fastcgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; + fastcgi_param SCRIPT_NAME git-http-backend; + fastcgi_param GIT_HTTP_EXPORT_ALL ""; + fastcgi_param GIT_PROJECT_ROOT {{ repo_service_home_folder }}/repo/openstackgit; + fastcgi_param PATH_INFO $1; + include /etc/nginx/fastcgi_params; + fastcgi_pass unix:/var/run/fcgiwrap.socket; + } +} diff --git a/templates/rsyncd.conf.j2 b/templates/rsyncd.conf.j2 new file mode 100644 index 0000000..12a2815 --- /dev/null +++ b/templates/rsyncd.conf.j2 @@ -0,0 +1,15 @@ +log file = /var/log/rsyncd.log +pid file = /var/run/rsyncd.pid +lock file = /var/run/rsync.lock + +max connections = 15 +timeout = 600 +transfer logging = true + +[openstack_mirror] +path = {{ repo_service_home_folder }}/repo +comment = OpenStack Private Cloud Mirror Server +use chroot = yes +read only = yes +uid = nobody +gid = nogroup diff --git a/tests/ansible-role-requirements.yml b/tests/ansible-role-requirements.yml new file mode 100644 index 0000000..314031c --- /dev/null +++ b/tests/ansible-role-requirements.yml @@ -0,0 +1,24 @@ +- name: apt_package_pinning + src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning + scm: git + version: master +- name: pip_install + src: https://git.openstack.org/openstack/openstack-ansible-pip_install + scm: git + version: master +- name: py_from_git + src: https://git.openstack.org/openstack/openstack-ansible-py_from_git + scm: git + version: master +- name: lxc_hosts + src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts + scm: git + version: master +- name: lxc_container_create + src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create + scm: git + version: master +- name: memcached_server + src: https://github.com/os-cloud/openstack-ansible-memcached_server + scm: git + version: master diff --git a/tests/ansible.cfg b/tests/ansible.cfg new file mode 100644 index 0000000..5c1deec --- /dev/null +++ b/tests/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +roles_path = ../../ +remote_tmp = ../.ansible/tmp/ +host_key_checking = False diff --git a/tests/inventory b/tests/inventory new file mode 100644 index 0000000..6c0833a --- /dev/null +++ b/tests/inventory @@ -0,0 +1,2 @@ +[all] +localhost ansible_connection=local ansible_become=True diff --git a/tests/test.yml b/tests/test.yml new file mode 100644 index 0000000..28e837a --- /dev/null +++ b/tests/test.yml @@ -0,0 +1,128 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Playbook for pre-role testing 1of2 + hosts: localhost + connection: local + pre_tasks: + - name: Ensure root ssh key + user: + name: "{{ ansible_env.USER | default('root') }}" + generate_ssh_key: "yes" + ssh_key_bits: 2048 + ssh_key_file: ".ssh/id_rsa" + - name: get the calling users key + command: cat ~/.ssh/id_rsa.pub + register: key_get + - set_fact: + lxc_container_ssh_key: "{{ key_get.stdout }}" + roles: + - role: "lxc_hosts" + lxc_net_address: 10.100.100.1 + lxc_net_dhcp_range: 10.100.100.2,10.100.100.253 + lxc_net_bridge: lxcbr0 + lxc_kernel_options: + - { key: 'fs.inotify.max_user_instances', value: 1024 } + lxc_container_caches: + - url: "https://rpc-repo.rackspace.com/container_images/rpc-trusty-container.tgz" + name: "trusty.tgz" + sha256sum: "56c6a6e132ea7d10be2f3e8104f47136ccf408b30e362133f0dc4a0a9adb4d0c" + chroot_path: trusty/rootfs-amd64 + # The $HOME directory is mocked to work with tox + # by defining the 'ansible_env' hash. This should + # NEVER be done outside of testing. + ansible_env: ## NEVER DO THIS OUTSIDE OF TESTING + HOME: "/tmp" + - role: "py_from_git" + git_repo: "https://github.com/lxc/python2-lxc" + git_dest: "/opt/lxc_python2" + git_install_branch: "master" + post_tasks: + # THIS TASK IS ONLY BEING DONE BECAUSE THE TOX SHARED LXC LIB IS NOT USABLE ON A + # HOST MACHINE THAT MAY NOT HAVE ACCESS TO THE VENV. + - name: Ensure the lxc lib is on the host + command: /usr/local/bin/pip install /opt/lxc_python2 + # Inventory is being pre-loaded using a post tasks instead of through a dynamic + # inventory system. While this is not a usual method for deployment it's being + # done for functional testing. + - name: Create container hosts + add_host: + groups: "all,all_containers,repo_all" + hostname: "{{ item.name }}" + inventory_hostname: "{{ item.name }}" + ansible_ssh_host: "{{ item.address }}" + ansible_become: true + properties: + service_name: "{{ item.service }}" + container_networks: + management_address: + address: "{{ item.address }}" + bridge: "lxcbr0" + interface: "eth1" + netmask: "255.255.252.0" + type: "veth" + physical_host: localhost + container_name: "{{ item.name }}" + with_items: + - { name: "container1", service: "service1", address: "10.100.100.101" } + - { name: "container2", service: "service2", address: "10.100.100.102" } + - { name: "container3", service: "service3", address: "10.100.100.103" } + +- name: Playbook for pre-role testing 2of2 + hosts: all_containers + connection: local + gather_facts: false + roles: + - role: "lxc_container_create" + lxc_container_release: trusty + lxc_container_backing_store: dir + global_environment_variables: + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +- name: Playbook for role testing + hosts: repo_all + gather_facts: true + roles: + - role: "{{ rolename | basename }}" + post_tasks: + - name: Confirm containers were created + command: curl -D - 10.100.100.101:8181 + - name: Check nginx is running + shell: "ps auxf | grep nginx" + delegate_to: 10.100.100.101 + - name: Check lsync is running + shell: "ps auxf | grep lsync" + delegate_to: 10.100.100.101 + - name: Check repo directory exists + stat: + path: /var/www/repo + delegate_to: 10.100.100.101 + register: repo_dir1 + - name: Check repo directory exists + stat: + path: /var/www/repo + delegate_to: 10.100.100.102 + register: repo_dir2 + - name: Check repo directory exists + stat: + path: /var/www/repo + delegate_to: 10.100.100.103 + register: repo_dir3 + - name: Check role functions + assert: + that: + - "repo_dir1.stat.exists" + - "repo_dir2.stat.exists" + - "repo_dir3.stat.exists" \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6d3d2aa --- /dev/null +++ b/tox.ini @@ -0,0 +1,97 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = docs,pep8,bashate,ansible-syntax,ansible-lint + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/dev-requirements.txt +commands = + /usr/bin/find . -type f -name "*.pyc" -delete + ansible-galaxy install \ + --role-file=ansible-role-requirements.yml \ + --ignore-errors \ + --force + +[testenv:docs] +commands = python setup.py build_sphinx + +# environment used by the -infra templated docs job +[testenv:venv] +deps = -r{toxinidir}/dev-requirements.txt +commands = {posargs} + +# Run hacking/flake8 check for all python files +[testenv:pep8] +deps = flake8 +whitelist_externals = bash +commands = + bash -c "grep -Irl \ + -e '!/usr/bin/env python' \ + -e '!/bin/python' \ + -e '!/usr/bin/python' \ + --exclude-dir '.*' \ + --exclude-dir 'doc' \ + --exclude-dir '*.egg' \ + --exclude-dir '*.egg-info' \ + --exclude 'tox.ini' \ + --exclude '*.sh' \ + {toxinidir} | xargs flake8 --verbose" + +[flake8] +# Ignores the following rules due to how ansible modules work in general +# F403 'from ansible.module_utils.basic import *' used; unable to detect undefined names +# H303 No wildcard (*) import. +ignore=F403,H303 + +# Run bashate check for all bash scripts +# Ignores the following rules: +# E003: Indent not multiple of 4 (we prefer to use multiples of 2) +[testenv:bashate] +deps = bashate +whitelist_externals = bash +commands = + bash -c "grep -Irl \ + -e '!/usr/bin/env bash' \ + -e '!/bin/bash' \ + -e '!/bin/sh' \ + --exclude-dir '.*' \ + --exclude-dir '*.egg' \ + --exclude-dir '*.egg-info' \ + --exclude 'tox.ini' \ + {toxinidir} | xargs bashate --verbose --ignore=E003" + +[testenv:ansible-syntax] +changedir = tests +commands = + ansible-galaxy install \ + --role-file=ansible-role-requirements.yml \ + --ignore-errors \ + --force + ansible-playbook -i inventory \ + --syntax-check \ + --list-tasks \ + -e "rolename={toxinidir}" \ + test.yml + +[testenv:ansible-lint] +changedir = tests +commands = + ansible-galaxy install \ + --role-file=ansible-role-requirements.yml \ + --ignore-errors \ + --force + ansible-lint test.yml + +[testenv:ansible-functional] +changedir = tests +commands = + ansible-galaxy install \ + --role-file=ansible-role-requirements.yml \ + --ignore-errors \ + --force + ansible-playbook -i inventory \ + -e "rolename={toxinidir}" \ + test.yml