diff --git a/anvil/distros/rhel.py b/anvil/distros/rhel.py index d9fa68ef..4356a12c 100644 --- a/anvil/distros/rhel.py +++ b/anvil/distros/rhel.py @@ -35,6 +35,7 @@ from anvil.components import nova from anvil.components import rabbit from anvil.packaging import yum +from anvil.packaging.helpers import changelog from anvil.components.helpers import nova as nhelper @@ -54,6 +55,11 @@ ResultActive=yes DEF_IDENT = 'unix-group:libvirtd' +def tar_it(to_where, what, wkdir): + tar_cmd = ['tar', '-cvzf', to_where, what] + return sh.execute(*tar_cmd, cwd=wkdir) + + class DBInstaller(db.DBInstaller): def _configure_db_confs(self): @@ -204,10 +210,10 @@ class DependencyPackager(comp.Component): comp.Component.__init__(self, *args, **kargs) self.package_dir = sh.joinpths(self.get_option('component_dir'), 'package') self.build_paths = {} - for name in ['SOURCES', 'SPECS', 'SRPMS', 'RPMS', 'BUILD']: + for name in ['sources', 'specs', 'srpms', 'rpms', 'build']: # Remove any old packaging directories... - sh.deldir(sh.joinpths(self.package_dir, name), True) - self.build_paths[name] = sh.mkdir(sh.joinpths(self.package_dir, name)) + sh.deldir(sh.joinpths(self.package_dir, name.upper()), True) + self.build_paths[name] = sh.mkdir(sh.joinpths(self.package_dir, name.upper())) self._cached_details = None def _requirements(self): @@ -216,9 +222,6 @@ class DependencyPackager(comp.Component): 'build': self._build_requirements(), } - def _description(self): - return '' - @property def details(self): if self._cached_details is not None: @@ -226,27 +229,16 @@ class DependencyPackager(comp.Component): self._cached_details = { 'name': self.name, 'version': 0, - 'release': 1, + 'release': self.get_int_option('release', default_value=1), 'packager': "%s <%s@%s>" % (sh.getuser(), sh.getuser(), sh.hostname()), 'changelog': '', 'license': 'Apache License, Version 2.0', 'automatic_dependencies': True, 'vendor': None, + 'url': '', + 'description': '', + 'summary': 'Package build of %s on %s' % (self.name, utils.rcf8222date()), } - # RPM apparently rejects descriptions with blank lines (even between content) - descr = self._description() - descr_lines = [] - for line in descr.splitlines(): - sline = line.strip() - if not sline: - continue - else: - descr_lines.append(line) - self._cached_details['description'] = "\n".join(descr_lines) - self._cached_details['summary'] = "\n".join(descr_lines[0:1]) - if not self._cached_details['summary']: - summary = 'Package build of %s on %s' % (self.name, utils.rcf8222date()) - self._cached_details['summary'] = summary return self._cached_details def _build_details(self): @@ -291,8 +283,9 @@ class DependencyPackager(comp.Component): return [] def _create_package(self): + files = self._gather_files() params = { - 'files': self._gather_files(), + 'files': files, 'requires': self._requirements(), 'obsoletes': self._obsoletes(), 'conflicts': self._conflicts(), @@ -304,10 +297,14 @@ class DependencyPackager(comp.Component): 'details': self.details, } (_fn, content) = utils.load_template('packaging', 'spec.tmpl') - spec_fn = sh.joinpths(self.build_paths['SPECS'], self._make_fn("spec")) + spec_base = self._make_fn("spec") + spec_fn = sh.joinpths(self.build_paths['specs'], spec_base) LOG.debug("Creating spec file %s with params:", spec_fn) + files['sources'].append("%s.tar.gz" % (spec_base)) utils.log_object(params, logger=LOG, level=logging.DEBUG) sh.write_file(spec_fn, utils.expand_template(content, params)) + tar_it(sh.joinpths(self.build_paths['sources'], "%s.tar.gz" % (spec_base)), + spec_base, wkdir=self.build_paths['specs']) def _build_requirements(self): return [] @@ -332,7 +329,8 @@ class DependencyPackager(comp.Component): class PythonPackager(DependencyPackager): def __init__(self, *args, **kargs): DependencyPackager.__init__(self, *args, **kargs) - self._details_adjusted = False + self._extended_details = None + self._setup_fn = sh.joinpths(self.get_option('app_dir'), 'setup.py') def _build_requirements(self): return [ @@ -342,46 +340,95 @@ class PythonPackager(DependencyPackager): 'python-setuptools', ] - def _make_source_archive(self): - return None + def _build_changelog(self): + try: + ch = changelog.RpmChangeLog(self.get_option('app_dir')) + return ch.formatLog() + except (excp.AnvilException, IOError): + return '' def _undefines(self): - to_undefine = DependencyPackager._undefines(self) - to_undefine.append('__check_files') - return to_undefine + undefine_what = DependencyPackager._undefines(self) + if self.get_bool_option('ignore_missing'): + undefine_what.append('__check_files') + return undefine_what + + def _gather_files(self): + files = DependencyPackager._gather_files(self) + files['directories'].append("%{python_sitelib}/" + (self.details['name'])) + files['files'].append("%{python_sitelib}/" + (self.details['name'])) + files['files'].append("%{python_sitelib}/" + "%s-*.egg-info/" % (self.details['name'])) + files['files'].append("%{_bindir}/") + return files + + def _build_details(self): + # See: http://www.rpm.org/max-rpm/s1-rpm-inside-macros.html + b_dets = DependencyPackager._build_details(self) + b_dets['setup'] = '-q -n %{name}-%{version}' + b_dets['action'] = '%{__python} setup.py build' + b_dets['install_how'] = '%{__python} setup.py install --prefix=%{_prefix} --root=%{buildroot}' + return b_dets + + def verify(self): + if not sh.isfile(self._setup_fn): + raise excp.PackageException(("Can not package %s since python" + " setup file at %s is missing") % (self.name, self._setup_fn)) + + def _make_source_archive(self): + with utils.tempdir() as td: + arch_base_name = "%s-%s" % (self.details['name'], self.details['version']) + sh.copytree(self.get_option('app_dir'), sh.joinpths(td, arch_base_name)) + arch_tmp_fn = sh.joinpths(td, "%s.tar.gz" % (arch_base_name)) + tar_it(arch_tmp_fn, arch_base_name, td) + sh.move(arch_tmp_fn, self.build_paths['sources']) + return "%s.tar.gz" % (arch_base_name) def _description(self): - app_dir = self.get_option('app_dir') - if not sh.isfile(sh.joinpths(app_dir, 'setup.py')): - return DependencyPackager._description(self) - describe_cmd = ['python', sh.joinpths(app_dir, 'setup.py'), '--description'] - (stdout, _stderr) = sh.execute(*describe_cmd, run_as_root=True, cwd=app_dir) - return stdout.strip() + describe_cmd = ['python', self._setup_fn, '--description'] + (stdout, _stderr) = sh.execute(*describe_cmd, run_as_root=True, cwd=self.get_option('app_dir')) + stdout = stdout.strip() + if stdout: + # RPM apparently rejects descriptions with blank lines (even between content) + descr_lines = [] + for line in stdout.splitlines(): + sline = line.strip() + if not sline: + continue + else: + descr_lines.append(line) + return descr_lines + return [] @property def details(self): base = super(PythonPackager, self).details - if self._details_adjusted: - return base - app_dir = self.get_option('app_dir') - if not sh.isfile(sh.joinpths(app_dir, 'setup.py')): - self._details_adjusted = True - return base - base_setup_cmd = ['python', sh.joinpths(app_dir, 'setup.py')] - replacements = { - 'version': '--version', - 'license': '--license', - 'name': '--name', - 'vendor': '--author', - } - for (key, opt) in replacements.items(): - cmd = base_setup_cmd + [opt] - (stdout, _stderr) = sh.execute(*cmd, run_as_root=True, cwd=app_dir) - stdout = stdout.strip() - if stdout: - base[key] = stdout - self._details_adjusted = True - return base + if self._extended_details is None: + ext_dets = { + 'automatic_dependencies': False, + } + setup_cmd = ['python', self._setup_fn] + replacements = { + 'version': '--version', + 'license': '--license', + 'name': '--name', + 'vendor': '--author', + 'url': '--url', + } + for (key, opt) in replacements.items(): + cmd = setup_cmd + [opt] + (stdout, _stderr) = sh.execute(*cmd, run_as_root=True, cwd=self.get_option('app_dir')) + stdout = stdout.strip() + if stdout: + ext_dets[key] = stdout + description = self._description() + if description: + ext_dets['description'] = "\n".join(description) + ext_dets['summary'] = utils.truncate_text("\n".join(description[0:1]), 50) + ext_dets['changelog'] = self._build_changelog() + self._extended_details = ext_dets + extended_dets = dict(base) + extended_dets.update(self._extended_details) + return extended_dets def package(self): i_sibling = self.siblings.get('install') @@ -391,6 +438,4 @@ class PythonPackager(DependencyPackager): if pips: for pip_info in pips: LOG.warn("Unable to package pip %s dependency in an rpm.", colorizer.quote(pip_info['name'])) - if not sh.isdir(self.get_option('app_dir')): - raise excp.PackageException("Can not package component %s without an application directory" % (self.name)) return DependencyPackager.package(self) diff --git a/anvil/packaging/helpers/changelog.py b/anvil/packaging/helpers/changelog.py new file mode 100644 index 00000000..0b098cb8 --- /dev/null +++ b/anvil/packaging/helpers/changelog.py @@ -0,0 +1,108 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Based off of http://www.brianlane.com/nice-changelog-entries.html +# +# git-changelog - Output a rpm changelog +# +# Copyright (C) 2009-2010 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +# +# Author: David Cantrell +# Author: Brian C. Lane + +import iso8601 +import textwrap + +from anvil import shell as sh + + +class RpmChangeLog(object): + def __init__(self, wkdir, max_history=100): + self.wkdir = wkdir + self.max_history = max_history + self.date_buckets = None + + def _getCommitDetail(self, commit, field): + detail_cmd = ['git', 'log', '-1', "--pretty=format:%s" % field, commit] + (stdout, stderr) = sh.execute(*detail_cmd, cwd=self.wkdir) + ret = stdout.strip('\n').splitlines() + if len(ret) == 1: + ret = ret[0] + else: + ret = filter(lambda x: x != '', ret) + return ret + + def _filter_logs(self, line): + if not line.strip(): + return False + if (line.find('l10n: ') != 41 and line.find('Merge commit') != 41 and line.find('Merge branch') != 41): + return True + return False + + def _getLog(self): + log_cmd = ['git', 'log', '--pretty=oneline', '-n%s' % (self.max_history)] + (sysout, stderr) = sh.execute(*log_cmd, cwd=self.wkdir) + lines = filter(self._filter_logs, sysout.strip('\n').splitlines()) + + # Extract the raw commit details + log = [] + for line in lines: + fields = line.split(' ') + commit = fields[0] + + # http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/pretty-formats.txt + summary = self._getCommitDetail(commit, "%s") + date = self._getCommitDetail(commit, "%ai") + author_email = self._getCommitDetail(commit, "%aE") + author_name = self._getCommitDetail(commit, "%an") + log.append({ + 'summary': summary, + 'when': iso8601.parse_date(date), + 'author_email': author_email, + 'author_name': author_name, + }) + + # Bucketize the dates + date_buckets = {} + for entry in log: + day = entry['when'].date() + if day in date_buckets: + date_buckets[day].append(entry) + else: + date_buckets[day] = [entry] + return date_buckets + + def formatLog(self): + if self.date_buckets is None: + self.date_buckets = self._getLog() + date_buckets = self.date_buckets + lines = [] + dates = date_buckets.keys() + for d in reversed(sorted(dates)): + summaries = date_buckets[d] + for msg in summaries: + header = "* %s %s <%s>" % (d.strftime("%a %b %d %Y"), + msg['author_name'], msg['author_email']) + lines.append(header) + summary = msg['summary'] + sublines = textwrap.wrap(summary, 77) + lines.append("- %s" % sublines[0]) + if len(sublines) > 1: + for subline in sublines[1:]: + lines.append(" %s" % subline) + # Replace utf8 with ? just incase + contents = "\n".join(lines) + contents = contents.decode('utf8').encode('ascii', 'replace') + return contents diff --git a/anvil/shell.py b/anvil/shell.py index 628236b2..10259839 100644 --- a/anvil/shell.py +++ b/anvil/shell.py @@ -210,7 +210,7 @@ def hostname(): try: return socket.gethostname() except socket.error: - return None + return 'localhost' def isuseable(path, options=os.W_OK | os.R_OK | os.X_OK): @@ -641,6 +641,13 @@ def copy(src, dst): return dst +def copytree(src, dst): + LOG.debug("Copying full tree: %r => %r" % (src, dst)) + if not is_dry_run(): + shutil.copytree(src, dst) + return dst + + def move(src, dst): LOG.debug("Moving: %r => %r" % (src, dst)) if not is_dry_run(): diff --git a/conf/distros/rhel.yaml b/conf/distros/rhel.yaml index 933eda09..29860546 100644 --- a/conf/distros/rhel.yaml +++ b/conf/distros/rhel.yaml @@ -56,7 +56,7 @@ components: running: anvil.components.cinder_client:CinderClientRuntime uninstall: anvil.components.cinder_client:CinderClientUninstaller test: anvil.components:PythonTestingComponent - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager db: action_classes: install: anvil.distros.rhel:DBInstaller @@ -223,7 +223,7 @@ components: running: anvil.components.glance:GlanceRuntime uninstall: anvil.components.glance:GlanceUninstaller test: anvil.components.glance:GlanceTester - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager # When parsing 'tools/pip-requires' and # 'tools/test-requires' (if they exist) # the following map will be used to translate names @@ -252,7 +252,7 @@ components: running: anvil.components.glance_client:GlanceClientRuntime uninstall: anvil.components.glance_client:GlanceClientUninstaller test: anvil.components.glance_client:GlanceClientTester - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager pips: - name: nosexcover - name: setuptools-git @@ -264,7 +264,7 @@ components: running: anvil.components.horizon:HorizonRuntime uninstall: anvil.components.horizon:HorizonUninstaller test: anvil.components:PythonTestingComponent - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager pip_to_package: - name: pytz package: @@ -288,7 +288,7 @@ components: running: anvil.components.keystone:KeystoneRuntime uninstall: anvil.components.keystone:KeystoneUninstaller test: anvil.components.keystone:KeystoneTester - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager packages: - name: MySQL-python pip_to_package: @@ -306,14 +306,14 @@ components: running: anvil.components.keystone_client:KeyStoneClientRuntime uninstall: anvil.components.keystone_client:KeyStoneClientUninstaller test: anvil.components:PythonTestingComponent - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager nova: action_classes: install: anvil.distros.rhel:NovaInstaller running: anvil.components.nova:NovaRuntime uninstall: anvil.components.nova:NovaUninstaller test: anvil.components.nova:NovaTester - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager packages: - name: MySQL-python - name: dnsmasq @@ -367,7 +367,7 @@ components: running: anvil.components.nova_client:NovaClientRuntime uninstall: anvil.components.nova_client:NovaClientUninstaller test: anvil.components:PythonTestingComponent - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager no-vnc: action_classes: install: anvil.components.novnc:NoVNCInstaller @@ -383,7 +383,7 @@ components: running: anvil.components.openstack_client:OpenStackClientRuntime uninstall: anvil.components.openstack_client:OpenStackClientUninstaller test: anvil.components.openstack_client:OpenStackClientTester - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager pips: - name: cliff pip_to_package: @@ -396,7 +396,7 @@ components: running: anvil.components.quantum_client:QuantumClientRuntime uninstall: anvil.components.quantum_client:QuantumClientUninstaller test: anvil.components:PythonTestingComponent - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager pip_to_package: - name: pyparsing package: @@ -409,7 +409,7 @@ components: running: anvil.distros.rhel:RabbitRuntime uninstall: anvil.components.rabbit:RabbitUninstaller test: anvil.components:EmptyTestingComponent - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:DependencyPackager packages: - name: rabbitmq-server pre-install: @@ -431,6 +431,6 @@ components: running: anvil.components.swift_client:SwiftClientRuntime uninstall: anvil.components.swift_client:SwiftClientUninstaller test: anvil.components:PythonTestingComponent - package: anvil.components:EmptyPackagingComponent + package: anvil.distros.rhel:PythonPackager ... diff --git a/conf/templates/packaging/spec.tmpl b/conf/templates/packaging/spec.tmpl index 66a5e4d6..553115c9 100644 --- a/conf/templates/packaging/spec.tmpl +++ b/conf/templates/packaging/spec.tmpl @@ -11,7 +11,7 @@ %define ${d} #end for #for $d in $undefines -#%undefine ${d} +%undefine ${d} #end for # # Spec file for $details.name auto-generated on ${date} by ${who} @@ -35,6 +35,10 @@ Name: $details.name Summary: $details.summary Version: $details.version Release: $details.release%{?dist} +Packager: $details.packager +#if $details.url +URL: $details.url +#end if #if $details.vendor Vendor: $details.vendor #end if @@ -89,19 +93,20 @@ $details.summary #end if %prep + #if $build.has_key('setup') %setup $build.setup #end if + #if $build.has_key('action') %build $build.action #end if %install -rm -rf %{buildroot} - -%clean -rm -rf %{buildroot} +#if $build.has_key('install_how') +$build.install_how +#end if %files %defattr(-,root,root,-) diff --git a/smithy b/smithy index 8899867a..f4e97be4 100755 --- a/smithy +++ b/smithy @@ -65,7 +65,9 @@ EOF return 1 fi echo "Installing needed distribution dependencies:" - yum install -y gcc git pylint python python-netifaces python-pep8 python-cheetah python-pip python-progressbar PyYAML python-ordereddict 2>&1 + pkgs="gcc git pylint python python-netifaces python-pep8 python-cheetah" + pkgs="$pkgs python-pip python-progressbar PyYAML python-ordereddict python-iso8601" + yum install -y $pkgs 2>&1 if [ $? -ne 0 ]; then return 1 fi @@ -82,7 +84,9 @@ bootstrap_ub() echo "Bootstrapping Ubuntu: $1" echo "Please wait..." echo "Installing needed distribution dependencies:" - apt-get install -y gcc git pep8 pylint python python-dev python-iniparse python-pip python-progressbar python-yaml python-cheetah + pkgs="gcc git pep8 pylint python python-dev python-iniparse" + pkgs="$pkgs python-pip python-progressbar python-yaml python-cheetah python-iso8601" + apt-get install -y $pkgs if [ $? -ne 0 ]; then return 1 fi