Refactor settings and project settings
The previous way that settings at both the global and project scope were being handled was pretty bad. This change seeks to make settings a bit more sane. Now we will pass in what we found in the manifest as the args to the specific settings constructor. Those that can be exposed directly will be, and those that cannot will be wrapped in @property decorators. In addition, moving the jinja templating from the manifest level to the individual project level. The reason is that it is not sufficient to just use templating globally - we need things like base_path's to be project context aware.
This commit is contained in:
parent
f0c3c08939
commit
dd5219dff8
@ -1,5 +1,8 @@
|
|||||||
# giftwrap changelog
|
# giftwrap changelog
|
||||||
|
|
||||||
|
## 0.3.0
|
||||||
|
- Refactor of how we handle settings both globally and at project level
|
||||||
|
|
||||||
## 0.2.0
|
## 0.2.0
|
||||||
- Adds colorization to the logging
|
- Adds colorization to the logging
|
||||||
- Adds a CHANGELOG
|
- Adds a CHANGELOG
|
||||||
|
@ -16,36 +16,21 @@
|
|||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from jinja2 import Template
|
|
||||||
|
|
||||||
from giftwrap.openstack_project import OpenstackProject
|
from giftwrap.openstack_project import OpenstackProject
|
||||||
from giftwrap.settings import Settings
|
from giftwrap.settings import Settings
|
||||||
|
|
||||||
|
|
||||||
class BuildSpec(object):
|
class BuildSpec(object):
|
||||||
def __init__(self, manifest, version=None, templatevars=None):
|
|
||||||
self._manifest = self._render_manifest(manifest, version, templatevars)
|
def __init__(self, manifest):
|
||||||
|
self._manifest = yaml.load(manifest)
|
||||||
|
self.settings = Settings.factory(self._manifest['settings'])
|
||||||
self.projects = self._render_projects()
|
self.projects = self._render_projects()
|
||||||
self.settings = self._render_settings()
|
|
||||||
|
|
||||||
def _render_manifest(self, manifest, version=None, templatevars=None):
|
|
||||||
manifestvars = {}
|
|
||||||
if templatevars:
|
|
||||||
manifestvars = yaml.load(templatevars)
|
|
||||||
|
|
||||||
if version:
|
|
||||||
manifestvars['version'] = version
|
|
||||||
|
|
||||||
template = Template(manifest)
|
|
||||||
manifest = template.render(manifestvars)
|
|
||||||
return yaml.load(manifest)
|
|
||||||
|
|
||||||
def _render_projects(self):
|
def _render_projects(self):
|
||||||
projects = []
|
projects = []
|
||||||
if 'projects' in self._manifest:
|
if 'projects' in self._manifest:
|
||||||
for project in self._manifest['projects']:
|
for project in self._manifest['projects']:
|
||||||
projects.append(OpenstackProject.factory(project))
|
projects.append(OpenstackProject.factory(self.settings,
|
||||||
|
project))
|
||||||
return projects
|
return projects
|
||||||
|
|
||||||
def _render_settings(self):
|
|
||||||
return Settings.factory(self._manifest)
|
|
||||||
|
@ -20,11 +20,14 @@ import sys
|
|||||||
from giftwrap import log
|
from giftwrap import log
|
||||||
from giftwrap.gerrit import GerritReview
|
from giftwrap.gerrit import GerritReview
|
||||||
from giftwrap.openstack_git_repo import OpenstackGitRepo
|
from giftwrap.openstack_git_repo import OpenstackGitRepo
|
||||||
|
from giftwrap.package import Package
|
||||||
|
from giftwrap.util import execute
|
||||||
|
|
||||||
LOG = log.get_logger()
|
LOG = log.get_logger()
|
||||||
|
|
||||||
|
|
||||||
class Builder(object):
|
class Builder(object):
|
||||||
|
|
||||||
def __init__(self, spec):
|
def __init__(self, spec):
|
||||||
self._spec = spec
|
self._spec = spec
|
||||||
|
|
||||||
@ -33,25 +36,33 @@ class Builder(object):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
spec = self._spec
|
spec = self._spec
|
||||||
base_path = spec.settings.base_path
|
|
||||||
# version = spec.settings.version
|
|
||||||
|
|
||||||
os.makedirs(base_path)
|
|
||||||
for project in self._spec.projects:
|
for project in self._spec.projects:
|
||||||
project_git_path = os.path.join(base_path, project.name)
|
LOG.info("Beginning to build '%s'", project.name)
|
||||||
repo = OpenstackGitRepo(project.giturl, project.ref)
|
os.makedirs(project.install_path)
|
||||||
repo.clone(project_git_path)
|
|
||||||
review = GerritReview(repo.change_id, project.project_path)
|
|
||||||
|
|
||||||
print "Cloned %s with change_id of: %s" % (project.name,
|
LOG.info("Fetching source code for '%s'", project.name)
|
||||||
repo.change_id)
|
repo = OpenstackGitRepo(project.giturl, project.gitref)
|
||||||
print "...with pip dependencies of:"
|
repo.clone(project.install_path)
|
||||||
print review.build_pip_dependencies(string=True)
|
review = GerritReview(repo.change_id, project.git_path)
|
||||||
|
|
||||||
|
LOG.info("Creating the virtualenv for '%s'", project.name)
|
||||||
|
execute(project.venv_command, project.install_path)
|
||||||
|
|
||||||
|
LOG.info("Installing '%s' pip dependencies to the virtualenv",
|
||||||
|
project.name)
|
||||||
|
execute(project.install_command %
|
||||||
|
review.build_pip_dependencies(string=True),
|
||||||
|
project.install_path)
|
||||||
|
|
||||||
|
LOG.info("Installing '%s' to the virtualenv", project.name)
|
||||||
|
execute(".venv/bin/python setup.py install",
|
||||||
|
project.install_path)
|
||||||
|
|
||||||
|
if not spec.settings.all_in_one:
|
||||||
|
pkg = Package(project.package_name, project.version,
|
||||||
|
project.install_path, True)
|
||||||
|
pkg.build()
|
||||||
|
|
||||||
# execute('python tools/install_venv.py', cwd=project_git_path)
|
|
||||||
# deps_string = " -d ".join(deps)
|
|
||||||
# execute("fpm -s dir -t deb -n foobar -d %s /tmp/openstack" %
|
|
||||||
# deps_string)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.exception("Oops. Something went wrong. Error was:\n%s", e)
|
LOG.exception("Oops. Something went wrong. Error was:\n%s", e)
|
||||||
sys.exit(-1)
|
sys.exit(-1)
|
||||||
|
@ -14,44 +14,108 @@
|
|||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import urlparse
|
||||||
|
|
||||||
OPENSTACK_PROJECTS = [
|
from jinja2 import Environment
|
||||||
{
|
|
||||||
'name': 'keystone',
|
DEFAULT_GITREF = 'master'
|
||||||
'project_path': 'openstack/keystone',
|
DEFAULT_GITURL = {
|
||||||
'giturl': 'https://github.com/openstack/keystone.git'
|
'openstack': 'https://git.openstack.org/openstack/',
|
||||||
|
'stackforge': 'https://github.com/stackforge/'
|
||||||
}
|
}
|
||||||
]
|
DEFAULT_VENV_COMMAND = "virtualenv .venv"
|
||||||
|
DEFAULT_INSTALL_COMMAND = ".venv/bin/pip install --extra-index http://pypi.openstack.org/openstack/ %s" # noqa
|
||||||
|
|
||||||
|
TEMPLATE_VARS = ('name', 'version', 'gitref', 'stackforge')
|
||||||
|
|
||||||
|
|
||||||
class OpenstackProject(object):
|
class OpenstackProject(object):
|
||||||
def __init__(self, name, ref=None, giturl=None):
|
|
||||||
if name not in OpenstackProject.get_project_names():
|
def __init__(self, settings, name, version=None, gitref=None, giturl=None,
|
||||||
raise Exception("'%s' is not a supported OpenStack project name" %
|
venv_command=None, install_command=None, install_path=None,
|
||||||
name)
|
package_name=None, stackforge=False):
|
||||||
|
self._settings = settings
|
||||||
self.name = name
|
self.name = name
|
||||||
self._project = [p for p in OPENSTACK_PROJECTS if p['name'] == name][0]
|
self._version = version
|
||||||
self.ref = ref if ref else 'master'
|
self._gitref = gitref
|
||||||
self.project_path = self._project['project_path']
|
self._giturl = giturl
|
||||||
if not giturl:
|
self._venv_command = venv_command
|
||||||
self.giturl = OpenstackProject.get_project_giturl(name)
|
self._install_command = install_command
|
||||||
else:
|
self._install_path = install_path
|
||||||
self.giturl = giturl
|
self._package_name = package_name
|
||||||
|
self.stackforge = stackforge
|
||||||
|
self._git_path = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def version(self):
|
||||||
|
if not self._version:
|
||||||
|
self._version = self._settings.version
|
||||||
|
return self._version
|
||||||
|
|
||||||
|
@property
|
||||||
|
def gitref(self):
|
||||||
|
if not self._gitref:
|
||||||
|
self._gitref = DEFAULT_GITREF
|
||||||
|
return self._gitref
|
||||||
|
|
||||||
|
@property
|
||||||
|
def giturl(self):
|
||||||
|
if not self._giturl:
|
||||||
|
key = 'openstack'
|
||||||
|
if self.stackforge:
|
||||||
|
key = 'stackforge'
|
||||||
|
self._giturl = urlparse.urljoin(DEFAULT_GITURL[key], self.name)
|
||||||
|
return self._giturl
|
||||||
|
|
||||||
|
@property
|
||||||
|
def venv_command(self):
|
||||||
|
if not self._venv_command:
|
||||||
|
self._venv_command = DEFAULT_VENV_COMMAND
|
||||||
|
return self._venv_command
|
||||||
|
|
||||||
|
@property
|
||||||
|
def package_name(self):
|
||||||
|
if not self._package_name:
|
||||||
|
self._package_name = \
|
||||||
|
self._render_from_settings('package_name_format')
|
||||||
|
return self._package_name
|
||||||
|
|
||||||
|
def _template_vars(self):
|
||||||
|
template_vars = {'project': self}
|
||||||
|
for var in TEMPLATE_VARS:
|
||||||
|
template_vars[var] = object.__getattribute__(self, var)
|
||||||
|
return template_vars
|
||||||
|
|
||||||
|
@property
|
||||||
|
def install_path(self):
|
||||||
|
if not self._install_path:
|
||||||
|
base_path = self._render_from_settings('base_path')
|
||||||
|
self._install_path = os.path.join(base_path, self.name)
|
||||||
|
return self._install_path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def install_command(self):
|
||||||
|
if not self._install_command:
|
||||||
|
self._install_command = DEFAULT_INSTALL_COMMAND
|
||||||
|
return self._install_command
|
||||||
|
|
||||||
|
@property
|
||||||
|
def git_path(self):
|
||||||
|
if not self._git_path:
|
||||||
|
gitorg = 'openstack'
|
||||||
|
if self.stackforge:
|
||||||
|
gitorg = 'stackforge'
|
||||||
|
self._git_path = '%s/%s' % (gitorg, self.name)
|
||||||
|
return self._git_path
|
||||||
|
|
||||||
|
def _render_from_settings(self, setting_name):
|
||||||
|
setting = getattr(self._settings, setting_name)
|
||||||
|
env = Environment()
|
||||||
|
env.add_extension('jinja2.ext.autoescape')
|
||||||
|
t = env.from_string(setting)
|
||||||
|
return t.render(self._template_vars())
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def factory(project_dict):
|
def factory(settings, project_dict):
|
||||||
name = project_dict.get('name', None)
|
return OpenstackProject(settings, **project_dict)
|
||||||
ref = project_dict.get('ref', None)
|
|
||||||
giturl = project_dict.get('giturl', None)
|
|
||||||
return OpenstackProject(name, ref, giturl)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_project_names():
|
|
||||||
return [n['name'] for n in OPENSTACK_PROJECTS]
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_project_giturl(name):
|
|
||||||
for project in OPENSTACK_PROJECTS:
|
|
||||||
if name == project['name']:
|
|
||||||
return project['giturl']
|
|
||||||
return None
|
|
||||||
|
@ -17,20 +17,34 @@
|
|||||||
|
|
||||||
class Settings(object):
|
class Settings(object):
|
||||||
|
|
||||||
DEFAULT_BASE_PATH = '/opt/openstack'
|
DEFAULTS = {
|
||||||
DEFAULT_VERSION = None
|
'package_name_format': 'openstack-{{ project.name }}',
|
||||||
|
'base_path': '/opt/openstack'
|
||||||
|
}
|
||||||
|
|
||||||
def __init__(self, version, base_path):
|
def __init__(self, package_name_format=None, version=None,
|
||||||
self._validate_settings(version, base_path)
|
base_path=None, all_in_one=False):
|
||||||
self.version = version
|
|
||||||
self.base_path = base_path
|
|
||||||
|
|
||||||
def _validate_settings(self, version, base_path):
|
|
||||||
if not version:
|
if not version:
|
||||||
raise Exception("You must provide a version")
|
raise Exception("'version' is a required settings")
|
||||||
|
self._package_name_format = package_name_format
|
||||||
|
self.version = version
|
||||||
|
self._base_path = base_path
|
||||||
|
self.all_in_one = all_in_one
|
||||||
|
|
||||||
|
@property
|
||||||
|
def package_name_format(self):
|
||||||
|
return self._get_setting('package_name_format')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def base_path(self):
|
||||||
|
return self._get_setting('base_path')
|
||||||
|
|
||||||
|
def _get_setting(self, setting_name):
|
||||||
|
setting = object.__getattribute__(self, '_%s' % setting_name)
|
||||||
|
if setting is None:
|
||||||
|
setting = Settings.DEFAULTS[setting_name]
|
||||||
|
return setting
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def factory(settings_dict):
|
def factory(settings_dict):
|
||||||
version = settings_dict.get('version', Settings.DEFAULT_VERSION)
|
return Settings(**settings_dict)
|
||||||
base_path = settings_dict.get('base_path', Settings.DEFAULT_BASE_PATH)
|
|
||||||
return Settings(version, base_path)
|
|
||||||
|
@ -28,16 +28,11 @@ def build(args):
|
|||||||
""" the entry point for all build subcommand tasks """
|
""" the entry point for all build subcommand tasks """
|
||||||
try:
|
try:
|
||||||
manifest = None
|
manifest = None
|
||||||
templatevars = None
|
|
||||||
|
|
||||||
with open(args.manifest, 'r') as fh:
|
with open(args.manifest, 'r') as fh:
|
||||||
manifest = fh.read()
|
manifest = fh.read()
|
||||||
|
|
||||||
if args.templatevars:
|
buildspec = BuildSpec(manifest)
|
||||||
with open(args.templatevars, 'r') as fh:
|
|
||||||
templatevars = fh.read()
|
|
||||||
|
|
||||||
buildspec = BuildSpec(manifest, args.version, templatevars)
|
|
||||||
builder = Builder(buildspec)
|
builder = Builder(buildspec)
|
||||||
builder.build()
|
builder.build()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@ -57,10 +52,6 @@ def main():
|
|||||||
build_subcmd = subparsers.add_parser('build',
|
build_subcmd = subparsers.add_parser('build',
|
||||||
description='build giftwrap packages')
|
description='build giftwrap packages')
|
||||||
build_subcmd.add_argument('-m', '--manifest', required=True)
|
build_subcmd.add_argument('-m', '--manifest', required=True)
|
||||||
build_subcmd.add_argument('-a', '--allinone', action='store_true')
|
|
||||||
build_subcmd.add_argument('-v', '--version')
|
|
||||||
build_subcmd.add_argument('-s', '--source', action='store_true')
|
|
||||||
build_subcmd.add_argument('-t', '--templatevars')
|
|
||||||
build_subcmd.set_defaults(func=build)
|
build_subcmd.set_defaults(func=build)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
@ -19,13 +19,21 @@ import unittest2 as unittest
|
|||||||
|
|
||||||
from giftwrap import settings
|
from giftwrap import settings
|
||||||
|
|
||||||
|
SAMPLE_SETTINGS = {
|
||||||
|
'package_name_format': 'my-package-name',
|
||||||
|
'version': '1.2',
|
||||||
|
'base_path': '/basepath'
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class TestSettings(unittest.TestCase):
|
class TestSettings(unittest.TestCase):
|
||||||
def test_factory(self):
|
def test_factory(self):
|
||||||
settings_dict = {'version': 'version', 'base_path': 'basepath'}
|
settings_dict = SAMPLE_SETTINGS
|
||||||
s = settings.Settings.factory(settings_dict)
|
s = settings.Settings.factory(settings_dict)
|
||||||
|
|
||||||
self.assertEquals('version', s.version)
|
self.assertEquals('my-package-name', s.package_name_format)
|
||||||
|
self.assertEquals('1.2', s.version)
|
||||||
|
self.assertEquals('/basepath', s.base_path)
|
||||||
|
|
||||||
def test_factory_has_default_base_path(self):
|
def test_factory_has_default_base_path(self):
|
||||||
settings_dict = {'version': 'version'}
|
settings_dict = {'version': 'version'}
|
||||||
@ -34,6 +42,8 @@ class TestSettings(unittest.TestCase):
|
|||||||
self.assertEquals('/opt/openstack', s.base_path)
|
self.assertEquals('/opt/openstack', s.base_path)
|
||||||
|
|
||||||
def test_factory_raises_when_version_missing(self):
|
def test_factory_raises_when_version_missing(self):
|
||||||
settings_dict = {'base_path': 'basepath'}
|
settings_dict = SAMPLE_SETTINGS
|
||||||
|
del(settings_dict['version'])
|
||||||
|
|
||||||
with self.assertRaises(Exception):
|
with self.assertRaises(Exception):
|
||||||
settings.Settings.factory(settings_dict)
|
settings.Settings.factory(settings_dict)
|
||||||
|
21
sample.yml
21
sample.yml
@ -1,7 +1,22 @@
|
|||||||
---
|
---
|
||||||
version: 1.2
|
settings:
|
||||||
base_path: '/tmp/openstack-test'
|
package_name_format: 'giftwrap-openstack-{{project.name}}'
|
||||||
|
version: '1.0-icehouse'
|
||||||
|
base_path: '/opt/giftwrap/openstack-{{version}}/'
|
||||||
|
|
||||||
projects:
|
projects:
|
||||||
- name: keystone
|
- name: keystone
|
||||||
ref: stable/havana
|
gitref: stable/icehouse
|
||||||
|
- name: glance
|
||||||
|
gitref: stable/icehouse
|
||||||
|
- name: nova
|
||||||
|
gitref: stable/icehouse
|
||||||
|
- name: cinder
|
||||||
|
gitref: stable/icehouse
|
||||||
|
- name: horizon
|
||||||
|
gitref: stable/icehouse
|
||||||
|
|
||||||
|
- name: python-keystoneclient
|
||||||
|
- name: python-glanceclient
|
||||||
|
- name: python-novaclient
|
||||||
|
- name: python-cinderclient
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[metadata]
|
[metadata]
|
||||||
name = giftwrap
|
name = giftwrap
|
||||||
version = 0.2.0
|
version = 0.3.0
|
||||||
summary = giftwrap - A tool to build full-stack native system packages.
|
summary = giftwrap - A tool to build full-stack native system packages.
|
||||||
description-file =
|
description-file =
|
||||||
README.md
|
README.md
|
||||||
|
Loading…
Reference in New Issue
Block a user