From dd5219dff83751f33b988833654a16391349ce71 Mon Sep 17 00:00:00 2001 From: Craig Tracey Date: Thu, 24 Jul 2014 16:06:25 -0400 Subject: [PATCH] Refactor settings and project settings The previous way that settings at both the global and project scope were being handled was pretty bad. This change seeks to make settings a bit more sane. Now we will pass in what we found in the manifest as the args to the specific settings constructor. Those that can be exposed directly will be, and those that cannot will be wrapped in @property decorators. In addition, moving the jinja templating from the manifest level to the individual project level. The reason is that it is not sufficient to just use templating globally - we need things like base_path's to be project context aware. --- CHANGELOG.md | 3 + giftwrap/build_spec.py | 27 ++----- giftwrap/builder.py | 43 +++++++---- giftwrap/openstack_project.py | 132 ++++++++++++++++++++++++-------- giftwrap/settings.py | 38 ++++++--- giftwrap/shell.py | 11 +-- giftwrap/tests/test_settings.py | 16 +++- sample.yml | 21 ++++- setup.cfg | 2 +- 9 files changed, 193 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17cdef2..63b73a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # giftwrap changelog +## 0.3.0 +- Refactor of how we handle settings both globally and at project level + ## 0.2.0 - Adds colorization to the logging - Adds a CHANGELOG diff --git a/giftwrap/build_spec.py b/giftwrap/build_spec.py index 6694b44..b460b32 100644 --- a/giftwrap/build_spec.py +++ b/giftwrap/build_spec.py @@ -16,36 +16,21 @@ import yaml -from jinja2 import Template - from giftwrap.openstack_project import OpenstackProject from giftwrap.settings import Settings class BuildSpec(object): - def __init__(self, manifest, version=None, templatevars=None): - self._manifest = self._render_manifest(manifest, version, templatevars) + + def __init__(self, manifest): + self._manifest = yaml.load(manifest) + self.settings = Settings.factory(self._manifest['settings']) self.projects = self._render_projects() - self.settings = self._render_settings() - - def _render_manifest(self, manifest, version=None, templatevars=None): - manifestvars = {} - if templatevars: - manifestvars = yaml.load(templatevars) - - if version: - manifestvars['version'] = version - - template = Template(manifest) - manifest = template.render(manifestvars) - return yaml.load(manifest) def _render_projects(self): projects = [] if 'projects' in self._manifest: for project in self._manifest['projects']: - projects.append(OpenstackProject.factory(project)) + projects.append(OpenstackProject.factory(self.settings, + project)) return projects - - def _render_settings(self): - return Settings.factory(self._manifest) diff --git a/giftwrap/builder.py b/giftwrap/builder.py index 9a6bfda..0d97d27 100644 --- a/giftwrap/builder.py +++ b/giftwrap/builder.py @@ -20,11 +20,14 @@ import sys from giftwrap import log from giftwrap.gerrit import GerritReview from giftwrap.openstack_git_repo import OpenstackGitRepo +from giftwrap.package import Package +from giftwrap.util import execute LOG = log.get_logger() class Builder(object): + def __init__(self, spec): self._spec = spec @@ -33,25 +36,33 @@ class Builder(object): try: spec = self._spec - base_path = spec.settings.base_path - # version = spec.settings.version - - os.makedirs(base_path) for project in self._spec.projects: - project_git_path = os.path.join(base_path, project.name) - repo = OpenstackGitRepo(project.giturl, project.ref) - repo.clone(project_git_path) - review = GerritReview(repo.change_id, project.project_path) + LOG.info("Beginning to build '%s'", project.name) + os.makedirs(project.install_path) - print "Cloned %s with change_id of: %s" % (project.name, - repo.change_id) - print "...with pip dependencies of:" - print review.build_pip_dependencies(string=True) + LOG.info("Fetching source code for '%s'", project.name) + repo = OpenstackGitRepo(project.giturl, project.gitref) + repo.clone(project.install_path) + review = GerritReview(repo.change_id, project.git_path) + + LOG.info("Creating the virtualenv for '%s'", project.name) + execute(project.venv_command, project.install_path) + + LOG.info("Installing '%s' pip dependencies to the virtualenv", + project.name) + execute(project.install_command % + review.build_pip_dependencies(string=True), + project.install_path) + + LOG.info("Installing '%s' to the virtualenv", project.name) + execute(".venv/bin/python setup.py install", + project.install_path) + + if not spec.settings.all_in_one: + pkg = Package(project.package_name, project.version, + project.install_path, True) + pkg.build() - # execute('python tools/install_venv.py', cwd=project_git_path) - # deps_string = " -d ".join(deps) - # execute("fpm -s dir -t deb -n foobar -d %s /tmp/openstack" % - # deps_string) except Exception as e: LOG.exception("Oops. Something went wrong. Error was:\n%s", e) sys.exit(-1) diff --git a/giftwrap/openstack_project.py b/giftwrap/openstack_project.py index 619b78c..2303622 100644 --- a/giftwrap/openstack_project.py +++ b/giftwrap/openstack_project.py @@ -14,44 +14,108 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations +import os +import urlparse -OPENSTACK_PROJECTS = [ - { - 'name': 'keystone', - 'project_path': 'openstack/keystone', - 'giturl': 'https://github.com/openstack/keystone.git' - } -] +from jinja2 import Environment + +DEFAULT_GITREF = 'master' +DEFAULT_GITURL = { + 'openstack': 'https://git.openstack.org/openstack/', + 'stackforge': 'https://github.com/stackforge/' +} +DEFAULT_VENV_COMMAND = "virtualenv .venv" +DEFAULT_INSTALL_COMMAND = ".venv/bin/pip install --extra-index http://pypi.openstack.org/openstack/ %s" # noqa + +TEMPLATE_VARS = ('name', 'version', 'gitref', 'stackforge') class OpenstackProject(object): - def __init__(self, name, ref=None, giturl=None): - if name not in OpenstackProject.get_project_names(): - raise Exception("'%s' is not a supported OpenStack project name" % - name) + + def __init__(self, settings, name, version=None, gitref=None, giturl=None, + venv_command=None, install_command=None, install_path=None, + package_name=None, stackforge=False): + self._settings = settings self.name = name - self._project = [p for p in OPENSTACK_PROJECTS if p['name'] == name][0] - self.ref = ref if ref else 'master' - self.project_path = self._project['project_path'] - if not giturl: - self.giturl = OpenstackProject.get_project_giturl(name) - else: - self.giturl = giturl + self._version = version + self._gitref = gitref + self._giturl = giturl + self._venv_command = venv_command + self._install_command = install_command + self._install_path = install_path + self._package_name = package_name + self.stackforge = stackforge + self._git_path = None + + @property + def version(self): + if not self._version: + self._version = self._settings.version + return self._version + + @property + def gitref(self): + if not self._gitref: + self._gitref = DEFAULT_GITREF + return self._gitref + + @property + def giturl(self): + if not self._giturl: + key = 'openstack' + if self.stackforge: + key = 'stackforge' + self._giturl = urlparse.urljoin(DEFAULT_GITURL[key], self.name) + return self._giturl + + @property + def venv_command(self): + if not self._venv_command: + self._venv_command = DEFAULT_VENV_COMMAND + return self._venv_command + + @property + def package_name(self): + if not self._package_name: + self._package_name = \ + self._render_from_settings('package_name_format') + return self._package_name + + def _template_vars(self): + template_vars = {'project': self} + for var in TEMPLATE_VARS: + template_vars[var] = object.__getattribute__(self, var) + return template_vars + + @property + def install_path(self): + if not self._install_path: + base_path = self._render_from_settings('base_path') + self._install_path = os.path.join(base_path, self.name) + return self._install_path + + @property + def install_command(self): + if not self._install_command: + self._install_command = DEFAULT_INSTALL_COMMAND + return self._install_command + + @property + def git_path(self): + if not self._git_path: + gitorg = 'openstack' + if self.stackforge: + gitorg = 'stackforge' + self._git_path = '%s/%s' % (gitorg, self.name) + return self._git_path + + def _render_from_settings(self, setting_name): + setting = getattr(self._settings, setting_name) + env = Environment() + env.add_extension('jinja2.ext.autoescape') + t = env.from_string(setting) + return t.render(self._template_vars()) @staticmethod - def factory(project_dict): - name = project_dict.get('name', None) - ref = project_dict.get('ref', None) - giturl = project_dict.get('giturl', None) - return OpenstackProject(name, ref, giturl) - - @staticmethod - def get_project_names(): - return [n['name'] for n in OPENSTACK_PROJECTS] - - @staticmethod - def get_project_giturl(name): - for project in OPENSTACK_PROJECTS: - if name == project['name']: - return project['giturl'] - return None + def factory(settings, project_dict): + return OpenstackProject(settings, **project_dict) diff --git a/giftwrap/settings.py b/giftwrap/settings.py index 16ff929..612a42b 100644 --- a/giftwrap/settings.py +++ b/giftwrap/settings.py @@ -17,20 +17,34 @@ class Settings(object): - DEFAULT_BASE_PATH = '/opt/openstack' - DEFAULT_VERSION = None + DEFAULTS = { + 'package_name_format': 'openstack-{{ project.name }}', + 'base_path': '/opt/openstack' + } - def __init__(self, version, base_path): - self._validate_settings(version, base_path) - self.version = version - self.base_path = base_path - - def _validate_settings(self, version, base_path): + def __init__(self, package_name_format=None, version=None, + base_path=None, all_in_one=False): if not version: - raise Exception("You must provide a version") + raise Exception("'version' is a required settings") + self._package_name_format = package_name_format + self.version = version + self._base_path = base_path + self.all_in_one = all_in_one + + @property + def package_name_format(self): + return self._get_setting('package_name_format') + + @property + def base_path(self): + return self._get_setting('base_path') + + def _get_setting(self, setting_name): + setting = object.__getattribute__(self, '_%s' % setting_name) + if setting is None: + setting = Settings.DEFAULTS[setting_name] + return setting @staticmethod def factory(settings_dict): - version = settings_dict.get('version', Settings.DEFAULT_VERSION) - base_path = settings_dict.get('base_path', Settings.DEFAULT_BASE_PATH) - return Settings(version, base_path) + return Settings(**settings_dict) diff --git a/giftwrap/shell.py b/giftwrap/shell.py index a0d4745..d686681 100644 --- a/giftwrap/shell.py +++ b/giftwrap/shell.py @@ -28,16 +28,11 @@ def build(args): """ the entry point for all build subcommand tasks """ try: manifest = None - templatevars = None with open(args.manifest, 'r') as fh: manifest = fh.read() - if args.templatevars: - with open(args.templatevars, 'r') as fh: - templatevars = fh.read() - - buildspec = BuildSpec(manifest, args.version, templatevars) + buildspec = BuildSpec(manifest) builder = Builder(buildspec) builder.build() except Exception as e: @@ -57,10 +52,6 @@ def main(): build_subcmd = subparsers.add_parser('build', description='build giftwrap packages') build_subcmd.add_argument('-m', '--manifest', required=True) - build_subcmd.add_argument('-a', '--allinone', action='store_true') - build_subcmd.add_argument('-v', '--version') - build_subcmd.add_argument('-s', '--source', action='store_true') - build_subcmd.add_argument('-t', '--templatevars') build_subcmd.set_defaults(func=build) args = parser.parse_args() diff --git a/giftwrap/tests/test_settings.py b/giftwrap/tests/test_settings.py index 139ccbc..249b338 100644 --- a/giftwrap/tests/test_settings.py +++ b/giftwrap/tests/test_settings.py @@ -19,13 +19,21 @@ import unittest2 as unittest from giftwrap import settings +SAMPLE_SETTINGS = { + 'package_name_format': 'my-package-name', + 'version': '1.2', + 'base_path': '/basepath' +} + class TestSettings(unittest.TestCase): def test_factory(self): - settings_dict = {'version': 'version', 'base_path': 'basepath'} + settings_dict = SAMPLE_SETTINGS s = settings.Settings.factory(settings_dict) - self.assertEquals('version', s.version) + self.assertEquals('my-package-name', s.package_name_format) + self.assertEquals('1.2', s.version) + self.assertEquals('/basepath', s.base_path) def test_factory_has_default_base_path(self): settings_dict = {'version': 'version'} @@ -34,6 +42,8 @@ class TestSettings(unittest.TestCase): self.assertEquals('/opt/openstack', s.base_path) def test_factory_raises_when_version_missing(self): - settings_dict = {'base_path': 'basepath'} + settings_dict = SAMPLE_SETTINGS + del(settings_dict['version']) + with self.assertRaises(Exception): settings.Settings.factory(settings_dict) diff --git a/sample.yml b/sample.yml index da42f34..fc8b2db 100644 --- a/sample.yml +++ b/sample.yml @@ -1,7 +1,22 @@ --- -version: 1.2 -base_path: '/tmp/openstack-test' +settings: + package_name_format: 'giftwrap-openstack-{{project.name}}' + version: '1.0-icehouse' + base_path: '/opt/giftwrap/openstack-{{version}}/' projects: - name: keystone - ref: stable/havana + gitref: stable/icehouse + - name: glance + gitref: stable/icehouse + - name: nova + gitref: stable/icehouse + - name: cinder + gitref: stable/icehouse + - name: horizon + gitref: stable/icehouse + + - name: python-keystoneclient + - name: python-glanceclient + - name: python-novaclient + - name: python-cinderclient diff --git a/setup.cfg b/setup.cfg index 81b9ff6..8fb9043 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = giftwrap -version = 0.2.0 +version = 0.3.0 summary = giftwrap - A tool to build full-stack native system packages. description-file = README.md