From b17f72d88458414b0a445130c267158e24bfbf16 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 28 Mar 2013 14:00:47 -0700 Subject: [PATCH] Initial commit. Change-Id: I9636b219862579fba20a389541bfdc51d41fe96c --- .gitignore | 5 + ChangeLog | 13 + LICENSE | 202 ++++++++++++++ openstack-common.conf | 7 + setup.py | 61 ++++ statusbot/__init__.py | 0 statusbot/bot.py | 360 ++++++++++++++++++++++++ statusbot/openstack/__init__.py | 0 statusbot/openstack/common/__init__.py | 0 statusbot/openstack/common/setup.py | 367 +++++++++++++++++++++++++ statusbot/openstack/common/version.py | 94 +++++++ statusbot/version.py | 20 ++ tools/pip-requires | 3 + tools/test-requires | 2 + tox.ini | 23 ++ 15 files changed, 1157 insertions(+) create mode 100644 .gitignore create mode 100644 ChangeLog create mode 100644 LICENSE create mode 100644 openstack-common.conf create mode 100644 setup.py create mode 100644 statusbot/__init__.py create mode 100644 statusbot/bot.py create mode 100644 statusbot/openstack/__init__.py create mode 100644 statusbot/openstack/common/__init__.py create mode 100644 statusbot/openstack/common/setup.py create mode 100644 statusbot/openstack/common/version.py create mode 100644 statusbot/version.py create mode 100644 tools/pip-requires create mode 100644 tools/test-requires create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6eb5e0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.tox +build/* +*.pyc +*.egg-info/ +AUTHORS diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..a175c89 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,13 @@ +commit 5f337ae74abd2feca91c9c499d79f840b8f81009 +Author: James E. Blair +Date: Thu Mar 28 14:00:47 2013 -0700 + + Initial commit. + + Change-Id: I9636b219862579fba20a389541bfdc51d41fe96c + +commit 2dec3213c62a1c81ab3daae2fd1eade464f88dad +Author: Openstack Project Creator +Date: Thu Mar 28 20:04:19 2013 +0000 + + Added .gitreview \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..75b5248 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/openstack-common.conf b/openstack-common.conf new file mode 100644 index 0000000..144f79f --- /dev/null +++ b/openstack-common.conf @@ -0,0 +1,7 @@ +[DEFAULT] + +# The list of modules to copy from openstack-common +modules=setup,version + +# The base module to hold the copy of openstack.common +base=statusbot diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2e3fa87 --- /dev/null +++ b/setup.py @@ -0,0 +1,61 @@ +#! /usr/bin/env python + +# Copyright 2011, 2013 OpenStack Foundation +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import setuptools + + +from statusbot.openstack.common import setup +from statusbot.version import version_info as version + +requires = setup.parse_requirements() +tests_require = setup.parse_requirements(['tools/test-requires']) +depend_links = setup.parse_dependency_links() + + +def read_file(file_name): + return open(os.path.join(os.path.dirname(__file__), file_name)).read() + + +setuptools.setup( + name="statusbot", + version=version.version_string(), + author='OpenStack Foundation', + author_email='openstack-infra@lists.openstack.org', + description="Client library for OpenStack Nova API.", + license="Apache License, Version 2.0", + url="https://github.com/openstack-infra/statusbot", + packages=setuptools.find_packages(exclude=['tests', 'tests.*']), + include_package_data=True, + setup_requires=['setuptools_git>=0.4'], + cmdclass=setup.get_cmdclass(), + install_requires=requires, + dependency_links=depend_links, + tests_require=tests_require, + test_suite="nose.collector", + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python" + ], + entry_points={ + "console_scripts": ["statusbot = statusbot.bot:main"] + } +) diff --git a/statusbot/__init__.py b/statusbot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/statusbot/bot.py b/statusbot/bot.py new file mode 100644 index 0000000..433dde6 --- /dev/null +++ b/statusbot/bot.py @@ -0,0 +1,360 @@ +#! /usr/bin/env python + +# Copyright 2011, 2013 OpenStack Foundation +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The configuration file should look like: +""" +[ircbot] +nick=NICKNAME +pass=PASSWORD +server=irc.freenode.net +port=6667 +channels=foo,bar +nicks=alice,bob + +[wiki] +user=StatusBot +password=password +url=https://wiki.example.com/w/api.php +pageid=1781 +""" + +import argparse +import ConfigParser +import daemon +import irc.bot +import logging +import os +import threading +import time +import simplemediawiki +import datetime +import re + +try: + import daemon.pidlockfile + pid_file_module = daemon.pidlockfile +except: + # as of python-daemon 1.6 it doesn't bundle pidlockfile anymore + # instead it depends on lockfile-0.9.1 + import daemon.pidfile + pid_file_module = daemon.pidfile + + +class UpdateInterface(object): + def alert(self, msg=None): + raise NotImplementedError() + + def notice(self, msg=None): + raise NotImplementedError() + + def log(self, msg=None): + raise NotImplementedError() + + def ok(self, msg=None): + raise NotImplementedError() + + +class StatusPage(UpdateInterface): + alert_re = re.compile(r'{{CI Alert\|(.*?)}}') + item_re = re.compile(r'^\* (.*)$') + + def __init__(self, config): + self.url = config.get('wiki', 'url') + self.pageid = config.get('wiki', 'pageid') + self.username = config.get('wiki', 'username') + self.password = config.get('wiki', 'password') + self.current_alert = None + self.items = [] + + def alert(self, msg): + self.update(set_alert=True, msg=msg) + + def notice(self, msg): + self.update(msg=msg) + + def log(self, msg): + self.update(msg=msg) + + def ok(self, msg): + self.update(clear_alert=True, msg=msg) + + def update(self, set_alert=None, clear_alert=None, msg=None): + self.wiki = simplemediawiki.MediaWiki(self.url) + self.wiki.login(self.username, self.password) + self.load() + if set_alert: + self.setAlert(msg) + if clear_alert: + self.setAlert(None) + if msg: + self.addItem(msg) + self.save() + + def load(self): + self.current_alert = None + self.items = [] + data = self.wiki.call(dict(action='query', + prop='revisions', + rvprop='content', + pageids=self.pageid, + format='json')) + text = data['query']['pages'][str(self.pageid)]['revisions'][0]['*'] + for line in text.split('\n'): + m = self.alert_re.match(line) + if m: + self.current_alert = m.group(1) + m = self.item_re.match(line) + if m: + self.items.append(m.group(1)) + + def save(self): + text = '' + if self.current_alert: + text += '{{CI Alert|%s}}\n\n' % self.current_alert + for item in self.items: + text += '* %s\n' % item + + data = self.wiki.call(dict(action='query', + prop='info', + pageids=self.pageid, + intoken='edit')) + token = data['query']['pages'][str(self.pageid)]['edittoken'] + data = self.wiki.call(dict(action='edit', + pageid=self.pageid, + bot=True, + text=text, + token=token)) + + def addItem(self, item, ts=None): + if not ts: + ts = datetime.datetime.now() + text = '%s %s' % (ts.strftime("%Y-%m-%d %H:%M:%S UTC"), item) + self.items.insert(0, text) + + def setAlert(self, current_alert): + self.current_alert = current_alert + + +class StatusBot(irc.bot.SingleServerIRCBot): + log = logging.getLogger("statusbot.bot") + + def __init__(self, channels, nicks, publishers, + nickname, password, server, port=6667): + irc.bot.SingleServerIRCBot.__init__(self, + [(server, port)], + nickname, nickname) + self.channel_list = channels + self.nicks = nicks + self.nickname = nickname + self.password = password + self.identify_msg_cap = False + self.ignore_topics = True + self.topic_lock = threading.Lock() + self.topics = {} + self.publishers = publishers + + def on_nicknameinuse(self, c, e): + self.log.debug("Nickname in use, releasing") + c.nick(c.get_nickname() + "_") + c.privmsg("nickserv", "identify %s " % self.password) + c.privmsg("nickserv", "ghost %s %s" % (self.nickname, self.password)) + c.privmsg("nickserv", "release %s %s" % (self.nickname, self.password)) + time.sleep(1) + c.nick(self.nickname) + + def on_welcome(self, c, e): + self.identify_msg_cap = False + self.log.debug("Requesting identify-msg capability") + c.cap('REQ', 'identify-msg') + c.cap('END') + self.log.debug("Identifying to nickserv") + c.privmsg("nickserv", "identify %s " % self.password) + for channel in self.channel_list: + self.log.info("Joining %s" % channel) + c.join(channel) + + def on_cap(self, c, e): + self.log.debug("Received cap response %s" % repr(e.arguments)) + if e.arguments[0] == 'ACK' and 'identify-msg' in e.arguments[1]: + self.log.debug("identify-msg cap acked") + self.identify_msg_cap = True + + def on_pubmsg(self, c, e): + if not self.identify_msg_cap: + self.log.debug("Ignoring message because identify-msg " + "cap not enabled") + return + nick = e.source.split('!')[0] + auth = e.arguments[0][0] + msg = e.arguments[0][1:] + if not msg.startswith('#status'): + return + if auth != '+': + self.log.debug("Ignoring message from unauthenticated " + "user %s" % nick) + return + if nick not in self.nicks: + self.log.debug("Ignoring message from untrusted user %s" % nick) + return + try: + self.handle_command(nick, msg) + except: + self.log.exception("Exception handling command %s" % msg) + + def handle_command(self, nick, msg): + parts = msg.split() + command = parts[1].lower() + text = ' '.join(parts[2:]) + + if command == 'alert': + self.log.info("Processing alert from %s: %s" % (nick, text)) + self.set_all_topics(text) + self.broadcast('NOTICE: ' + text) + for p in self.publishers: + p.alert(text) + elif command == 'notice': + self.log.info("Processing notice from %s: %s" % (nick, text)) + self.broadcast('NOTICE: ' + text) + for p in self.publishers: + p.notice(text) + elif command == 'log': + self.log.info("Processing log from %s: %s" % (nick, text)) + for p in self.publishers: + p.log(text) + elif command == 'ok': + self.log.info("Processing ok from %s: %s" % (nick, text)) + self.restore_all_topics() + if text: + self.broadcast('NOTICE: ' + text) + for p in self.publishers: + p.ok(text) + else: + self.log.info("Unknown command %s from %s: %s" % ( + command, nick, msg)) + + def broadcast(self, msg): + for channel in self.channel_list: + self.send(channel, msg) + + def restore_all_topics(self): + t = threading.Thread(target=self._restore_all_topics, args=()) + t.start() + + def _restore_all_topics(self): + self.topic_lock.acquire() + try: + if self.topics: + for channel in self.channel_list: + self.set_topic(channel, self.topics[channel]) + self.topics = {} + finally: + self.topic_lock.release() + + def set_all_topics(self, topic): + t = threading.Thread(target=self._set_all_topics, args=(topic,)) + t.start() + + def _set_all_topics(self, topic): + self.topic_lock.acquire() + try: + if not self.topics: + self.save_topics() + for channel in self.channel_list: + self.set_topic(channel, topic) + finally: + self.topic_lock.release() + + def save_topics(self): + # Save all the current topics + self.ignore_topics = False + for channel in self.channel_list: + self.connection.topic(channel) + time.sleep(0.5) + start = time.time() + done = False + while time.time() < start + 300: + if len(self.topics) == len(self.channel_list): + done = True + break + time.sleep(0.5) + self.ignore_topics = True + if not done: + raise Exception("Unable to save topics") + + def on_currenttopic(self, c, e): + if self.ignore_topics: + return + self.topics[e.arguments[0]] = e.arguments[1] + + def send(self, channel, msg): + self.connection.privmsg(channel, msg) + time.sleep(0.5) + + def set_topic(self, channel, topic): + self.connection.topic(channel, topic) + self.connection.privmsg('ChanServ', 'topic %s %s' % (channel, topic)) + time.sleep(0.5) + + +def _main(configpath): + config = ConfigParser.ConfigParser() + config.read(configpath) + setup_logging(config) + + channels = ['#' + name.strip() for name in + config.get('ircbot', 'channels').split(',')] + nicks = [name.strip() for name in + config.get('ircbot', 'nicks').split(',')] + publishers = [StatusPage(config)] + + bot = StatusBot(channels, nicks, publishers, + config.get('ircbot', 'nick'), + config.get('ircbot', 'pass'), + config.get('ircbot', 'server'), + config.getint('ircbot', 'port')) + bot.start() + + +def main(): + parser = argparse.ArgumentParser(description='Status bot.') + parser.add_argument('-c', dest='config', nargs=1, + help='specify the config file') + parser.add_argument('-d', dest='nodaemon', action='store_true', + help='do not run as a daemon') + args = parser.parse_args() + + if not args.nodaemon: + pid = pid_file_module.TimeoutPIDLockFile( + "/var/run/statusbot/statusbot.pid", 10) + with daemon.DaemonContext(pidfile=pid): + _main(args.config) + _main(args.config) + + +def setup_logging(config): + if config.has_option('ircbot', 'log_config'): + log_config = config.get('ircbot', 'log_config') + fp = os.path.expanduser(log_config) + if not os.path.exists(fp): + raise Exception("Unable to read logging config file at %s" % fp) + logging.config.fileConfig(fp) + else: + logging.basicConfig(level=logging.DEBUG) + + +if __name__ == "__main__": + main() diff --git a/statusbot/openstack/__init__.py b/statusbot/openstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/statusbot/openstack/common/__init__.py b/statusbot/openstack/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/statusbot/openstack/common/setup.py b/statusbot/openstack/common/setup.py new file mode 100644 index 0000000..dec74fd --- /dev/null +++ b/statusbot/openstack/common/setup.py @@ -0,0 +1,367 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities with minimum-depends for use in setup.py +""" + +import email +import os +import re +import subprocess +import sys + +from setuptools.command import sdist + + +def parse_mailmap(mailmap='.mailmap'): + mapping = {} + if os.path.exists(mailmap): + with open(mailmap, 'r') as fp: + for l in fp: + try: + canonical_email, alias = re.match( + r'[^#]*?(<.+>).*(<.+>).*', l).groups() + except AttributeError: + continue + mapping[alias] = canonical_email + return mapping + + +def _parse_git_mailmap(git_dir, mailmap='.mailmap'): + mailmap = os.path.join(os.path.dirname(git_dir), mailmap) + return parse_mailmap(mailmap) + + +def canonicalize_emails(changelog, mapping): + """Takes in a string and an email alias mapping and replaces all + instances of the aliases in the string with their real email. + """ + for alias, email_address in mapping.iteritems(): + changelog = changelog.replace(alias, email_address) + return changelog + + +# Get requirements from the first file that exists +def get_reqs_from_files(requirements_files): + for requirements_file in requirements_files: + if os.path.exists(requirements_file): + with open(requirements_file, 'r') as fil: + return fil.read().split('\n') + return [] + + +def parse_requirements(requirements_files=['requirements.txt', + 'tools/pip-requires']): + requirements = [] + for line in get_reqs_from_files(requirements_files): + # For the requirements list, we need to inject only the portion + # after egg= so that distutils knows the package it's looking for + # such as: + # -e git://github.com/openstack/nova/master#egg=nova + if re.match(r'\s*-e\s+', line): + requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + line)) + # such as: + # http://github.com/openstack/nova/zipball/master#egg=nova + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here + elif re.match(r'\s*-f\s+', line): + pass + # argparse is part of the standard library starting with 2.7 + # adding it to the requirements list screws distro installs + elif line == 'argparse' and sys.version_info >= (2, 7): + pass + else: + requirements.append(line) + + return requirements + + +def parse_dependency_links(requirements_files=['requirements.txt', + 'tools/pip-requires']): + dependency_links = [] + # dependency_links inject alternate locations to find packages listed + # in requirements + for line in get_reqs_from_files(requirements_files): + # skip comments and blank lines + if re.match(r'(\s*#)|(\s*$)', line): + continue + # lines with -e or -f need the whole line, minus the flag + if re.match(r'\s*-[ef]\s+', line): + dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) + # lines that are only urls can go in unmolested + elif re.match(r'\s*https?:', line): + dependency_links.append(line) + return dependency_links + + +def _run_shell_command(cmd, throw_on_error=False): + if os.name == 'nt': + output = subprocess.Popen(["cmd.exe", "/C", cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + output = subprocess.Popen(["/bin/sh", "-c", cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out = output.communicate() + if output.returncode and throw_on_error: + raise Exception("%s returned %d" % cmd, output.returncode) + if len(out) == 0: + return None + if len(out[0].strip()) == 0: + return None + return out[0].strip() + + +def _get_git_directory(): + parent_dir = os.path.dirname(__file__) + while True: + git_dir = os.path.join(parent_dir, '.git') + if os.path.exists(git_dir): + return git_dir + parent_dir, child = os.path.split(parent_dir) + if not child: # reached to root dir + return None + + +def write_git_changelog(): + """Write a changelog based on the git changelog.""" + new_changelog = 'ChangeLog' + git_dir = _get_git_directory() + if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'): + if git_dir: + git_log_cmd = 'git --git-dir=%s log' % git_dir + changelog = _run_shell_command(git_log_cmd) + mailmap = _parse_git_mailmap(git_dir) + with open(new_changelog, "w") as changelog_file: + changelog_file.write(canonicalize_emails(changelog, mailmap)) + else: + open(new_changelog, 'w').close() + + +def generate_authors(): + """Create AUTHORS file using git commits.""" + jenkins_email = 'jenkins@review.(openstack|stackforge).org' + old_authors = 'AUTHORS.in' + new_authors = 'AUTHORS' + git_dir = _get_git_directory() + if not os.getenv('SKIP_GENERATE_AUTHORS'): + if git_dir: + # don't include jenkins email address in AUTHORS file + git_log_cmd = ("git --git-dir=" + git_dir + + " log --format='%aN <%aE>' | sort -u | " + "egrep -v '" + jenkins_email + "'") + changelog = _run_shell_command(git_log_cmd) + signed_cmd = ("git log --git-dir=" + git_dir + + " | grep -i Co-authored-by: | sort -u") + signed_entries = _run_shell_command(signed_cmd) + if signed_entries: + new_entries = "\n".join( + [signed.split(":", 1)[1].strip() + for signed in signed_entries.split("\n") if signed]) + changelog = "\n".join((changelog, new_entries)) + mailmap = _parse_git_mailmap(git_dir) + with open(new_authors, 'w') as new_authors_fh: + new_authors_fh.write(canonicalize_emails(changelog, mailmap)) + if os.path.exists(old_authors): + with open(old_authors, "r") as old_authors_fh: + new_authors_fh.write('\n' + old_authors_fh.read()) + else: + open(new_authors, 'w').close() + + +_rst_template = """%(heading)s +%(underline)s + +.. automodule:: %(module)s + :members: + :undoc-members: + :show-inheritance: +""" + + +def get_cmdclass(): + """Return dict of commands to run from setup.py.""" + + cmdclass = dict() + + def _find_modules(arg, dirname, files): + for filename in files: + if filename.endswith('.py') and filename != '__init__.py': + arg["%s.%s" % (dirname.replace('/', '.'), + filename[:-3])] = True + + class LocalSDist(sdist.sdist): + """Builds the ChangeLog and Authors files from VC first.""" + + def run(self): + write_git_changelog() + generate_authors() + # sdist.sdist is an old style class, can't use super() + sdist.sdist.run(self) + + cmdclass['sdist'] = LocalSDist + + # If Sphinx is installed on the box running setup.py, + # enable setup.py to build the documentation, otherwise, + # just ignore it + try: + from sphinx.setup_command import BuildDoc + + class LocalBuildDoc(BuildDoc): + + builders = ['html', 'man'] + + def generate_autoindex(self): + print "**Autodocumenting from %s" % os.path.abspath(os.curdir) + modules = {} + option_dict = self.distribution.get_option_dict('build_sphinx') + source_dir = os.path.join(option_dict['source_dir'][1], 'api') + if not os.path.exists(source_dir): + os.makedirs(source_dir) + for pkg in self.distribution.packages: + if '.' not in pkg: + os.path.walk(pkg, _find_modules, modules) + module_list = modules.keys() + module_list.sort() + autoindex_filename = os.path.join(source_dir, 'autoindex.rst') + with open(autoindex_filename, 'w') as autoindex: + autoindex.write(""".. toctree:: + :maxdepth: 1 + +""") + for module in module_list: + output_filename = os.path.join(source_dir, + "%s.rst" % module) + heading = "The :mod:`%s` Module" % module + underline = "=" * len(heading) + values = dict(module=module, heading=heading, + underline=underline) + + print "Generating %s" % output_filename + with open(output_filename, 'w') as output_file: + output_file.write(_rst_template % values) + autoindex.write(" %s.rst\n" % module) + + def run(self): + if not os.getenv('SPHINX_DEBUG'): + self.generate_autoindex() + + for builder in self.builders: + self.builder = builder + self.finalize_options() + self.project = self.distribution.get_name() + self.version = self.distribution.get_version() + self.release = self.distribution.get_version() + BuildDoc.run(self) + + class LocalBuildLatex(LocalBuildDoc): + builders = ['latex'] + + cmdclass['build_sphinx'] = LocalBuildDoc + cmdclass['build_sphinx_latex'] = LocalBuildLatex + except ImportError: + pass + + return cmdclass + + +def _get_revno(git_dir): + """Return the number of commits since the most recent tag. + + We use git-describe to find this out, but if there are no + tags then we fall back to counting commits since the beginning + of time. + """ + describe = _run_shell_command( + "git --git-dir=%s describe --always" % git_dir) + if "-" in describe: + return describe.rsplit("-", 2)[-2] + + # no tags found + revlist = _run_shell_command( + "git --git-dir=%s rev-list --abbrev-commit HEAD" % git_dir) + return len(revlist.splitlines()) + + +def _get_version_from_git(pre_version): + """Return a version which is equal to the tag that's on the current + revision if there is one, or tag plus number of additional revisions + if the current revision has no tag.""" + + git_dir = _get_git_directory() + if git_dir: + if pre_version: + try: + return _run_shell_command( + "git --git-dir=" + git_dir + " describe --exact-match", + throw_on_error=True).replace('-', '.') + except Exception: + sha = _run_shell_command( + "git --git-dir=" + git_dir + " log -n1 --pretty=format:%h") + return "%s.a%s.g%s" % (pre_version, _get_revno(git_dir), sha) + else: + return _run_shell_command( + "git --git-dir=" + git_dir + " describe --always").replace( + '-', '.') + return None + + +def _get_version_from_pkg_info(package_name): + """Get the version from PKG-INFO file if we can.""" + try: + pkg_info_file = open('PKG-INFO', 'r') + except (IOError, OSError): + return None + try: + pkg_info = email.message_from_file(pkg_info_file) + except email.MessageError: + return None + # Check to make sure we're in our own dir + if pkg_info.get('Name', None) != package_name: + return None + return pkg_info.get('Version', None) + + +def get_version(package_name, pre_version=None): + """Get the version of the project. First, try getting it from PKG-INFO, if + it exists. If it does, that means we're in a distribution tarball or that + install has happened. Otherwise, if there is no PKG-INFO file, pull the + version from git. + + We do not support setup.py version sanity in git archive tarballs, nor do + we support packagers directly sucking our git repo into theirs. We expect + that a source tarball be made from our git repo - or that if someone wants + to make a source tarball from a fork of our repo with additional tags in it + that they understand and desire the results of doing that. + """ + version = os.environ.get("OSLO_PACKAGE_VERSION", None) + if version: + return version + version = _get_version_from_pkg_info(package_name) + if version: + return version + version = _get_version_from_git(pre_version) + if version: + return version + raise Exception("Versioning for this project requires either an sdist" + " tarball, or access to an upstream git repository.") diff --git a/statusbot/openstack/common/version.py b/statusbot/openstack/common/version.py new file mode 100644 index 0000000..2fd0f9e --- /dev/null +++ b/statusbot/openstack/common/version.py @@ -0,0 +1,94 @@ + +# Copyright 2012 OpenStack Foundation +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities for consuming the version from pkg_resources. +""" + +import pkg_resources + + +class VersionInfo(object): + + def __init__(self, package): + """Object that understands versioning for a package + :param package: name of the python package, such as glance, or + python-glanceclient + """ + self.package = package + self.release = None + self.version = None + self._cached_version = None + + def __str__(self): + """Make the VersionInfo object behave like a string.""" + return self.version_string() + + def __repr__(self): + """Include the name.""" + return "VersionInfo(%s:%s)" % (self.package, self.version_string()) + + def _get_version_from_pkg_resources(self): + """Get the version of the package from the pkg_resources record + associated with the package.""" + try: + requirement = pkg_resources.Requirement.parse(self.package) + provider = pkg_resources.get_provider(requirement) + return provider.version + except pkg_resources.DistributionNotFound: + # The most likely cause for this is running tests in a tree + # produced from a tarball where the package itself has not been + # installed into anything. Revert to setup-time logic. + from statusbot.openstack.common import setup + return setup.get_version(self.package) + + def release_string(self): + """Return the full version of the package including suffixes indicating + VCS status. + """ + if self.release is None: + self.release = self._get_version_from_pkg_resources() + + return self.release + + def version_string(self): + """Return the short version minus any alpha/beta tags.""" + if self.version is None: + parts = [] + for part in self.release_string().split('.'): + if part[0].isdigit(): + parts.append(part) + else: + break + self.version = ".".join(parts) + + return self.version + + # Compatibility functions + canonical_version_string = version_string + version_string_with_vcs = release_string + + def cached_version_string(self, prefix=""): + """Generate an object which will expand in a string context to + the results of version_string(). We do this so that don't + call into pkg_resources every time we start up a program when + passing version information into the CONF constructor, but + rather only do the calculation when and if a version is requested + """ + if not self._cached_version: + self._cached_version = "%s%s" % (prefix, + self.version_string()) + return self._cached_version diff --git a/statusbot/version.py b/statusbot/version.py new file mode 100644 index 0000000..07b3440 --- /dev/null +++ b/statusbot/version.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011, 2013 OpenStack Foundation +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from statusbot.openstack.common import version as common_version + +version_info = common_version.VersionInfo('statusbot') diff --git a/tools/pip-requires b/tools/pip-requires new file mode 100644 index 0000000..6ef5e9a --- /dev/null +++ b/tools/pip-requires @@ -0,0 +1,3 @@ +simplemediawiki +irc +python-daemon diff --git a/tools/test-requires b/tools/test-requires new file mode 100644 index 0000000..4cdcb3a --- /dev/null +++ b/tools/test-requires @@ -0,0 +1,2 @@ +pep8==1.1 +pyflakes diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..bacec78 --- /dev/null +++ b/tox.ini @@ -0,0 +1,23 @@ +[tox] +envlist = pep8,pyflakes + +[testenv] +setenv = VIRTUAL_ENV={envdir} +sitepackages=True +deps = -r{toxinidir}/tools/pip-requires + -r{toxinidir}/tools/test-requires +commands = nosetests {posargs} + +[testenv:pep8] +deps = pep8==1.1 +commands = pep8 --repeat --show-source --exclude=.venv,.tox,dist,doc,*.egg . + +[testenv:cover] +setenv = NOSE_WITH_COVERAGE=1 + +[testenv:pyflakes] +deps = pyflakes +commands = pyflakes statusbot/bot.py statusbot/version.py setup.py + +[testenv:venv] +commands = {posargs}