diff --git a/giftwrap/builder.py b/giftwrap/builder.py deleted file mode 100644 index d9403cf..0000000 --- a/giftwrap/builder.py +++ /dev/null @@ -1,54 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2014, Craig Tracey -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations - -import logging - -LOG = logging.getLogger(__name__) - - -class Builder(object): - - def __init__(self, spec): - self._spec = spec - self.settings = spec.settings - - def _validate_settings(self): - raise NotImplementedError() - - def _build(self): - raise NotImplementedError() - - def _cleanup(self): - raise NotImplementedError() - - def build(self): - self._validate_settings() - self._build() - - def cleanup(self): - self._cleanup() - - -from giftwrap.builders.package_builder import PackageBuilder -from giftwrap.builders.docker_builder import DockerBuilder - - -def create_builder(spec): - if spec.settings.build_type == 'package': - return PackageBuilder(spec) - elif spec.settings.build_type == 'docker': - return DockerBuilder(spec) - raise Exception("Unknown build_type: '%s'", spec.settings.build_type) diff --git a/giftwrap/builders/__init__.py b/giftwrap/builders/__init__.py index e69de29..666507a 100644 --- a/giftwrap/builders/__init__.py +++ b/giftwrap/builders/__init__.py @@ -0,0 +1,157 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2014, Craig Tracey +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations + +import logging +import os + +from giftwrap.gerrit import GerritReview + +from abc import abstractmethod, ABCMeta + +LOG = logging.getLogger(__name__) + + +class Builder(object): + __metaclass__ = ABCMeta + + def __init__(self, spec): + self._temp_dir = None + self._temp_src_dir = None + self._spec = spec + + def _get_venv_pip_path(self, venv_path): + return os.path.join(venv_path, 'bin/pip') + + def _get_gerrit_dependencies(self, repo, project): + try: + review = GerritReview(repo.head.change_id, project.git_path) + return review.build_pip_dependencies(string=True) + except Exception as e: + LOG.warning("Could not install gerrit dependencies!!! " + "Error was: %s", e) + return "" + + def _build_project(self, project): + self._prepare_project_build(project) + self._make_dir(project.install_path) + + # clone the source + src_clone_dir = os.path.join(self._temp_src_dir, project.name) + repo = self._clone_project(project.giturl, project.name, + project.gitref, project.gitdepth, + src_clone_dir) + + # create and build the virtualenv + self._create_virtualenv(project.venv_command, project.install_path) + dependencies = "" + if project.pip_dependencies: + dependencies = " ".join(project.pip_dependencies) + if self._spec.settings.gerrit_dependencies: + dependencies = "%s %s" % (dependencies, + self._get_gerrit_dependencies(repo, + project)) + if len(dependencies): + self._install_pip_dependencies(project.install_path, dependencies) + + if self._spec.settings.include_config: + self._copy_sample_config(src_clone_dir, project) + + self._install_project(project.install_path, src_clone_dir) + + # finish up + self._finalize_project_build(project) + + def build(self): + spec = self._spec + + self._prepare_build() + + # Create a temporary directory for the source code + self._temp_dir = self._make_temp_dir() + self._temp_src_dir = os.path.join(self._temp_dir, 'src') + LOG.debug("Temporary working directory: %s", self._temp_dir) + + for project in spec.projects: + self._build_project(project) + + self._finalize_build() + + def cleanup(self): + self._cleanup_build() + + @abstractmethod + def _execute(self, command, cwd=None, exit=0): + return + + @abstractmethod + def _make_temp_dir(self, prefix='giftwrap'): + return + + @abstractmethod + def _make_dir(self, path, mode=0777): + return + + @abstractmethod + def _prepare_build(self): + return + + @abstractmethod + def _prepare_project_build(self, project): + return + + @abstractmethod + def _clone_project(self, project): + return + + @abstractmethod + def _create_virtualenv(self, venv_command, path): + return + + @abstractmethod + def _install_pip_dependencies(self, venv_path, dependencies): + return + + @abstractmethod + def _copy_sample_config(self, src_clone_dir, project): + return + + @abstractmethod + def _install_project(self, venv_path, src_clone_dir): + return + + @abstractmethod + def _finalize_project_build(self, project): + return + + @abstractmethod + def _finalize_build(self): + return + + @abstractmethod + def _cleanup_build(self): + return + + +from giftwrap.builders.package_builder import PackageBuilder # noqa +from giftwrap.builders.docker_builder import DockerBuilder # noqa + + +class BuilderFactory: + + @staticmethod + def create_builder(builder_type, build_spec): + targetclass = "%sBuilder" % builder_type.capitalize() + return globals()[targetclass](build_spec) diff --git a/giftwrap/builders/docker_builder.py b/giftwrap/builders/docker_builder.py index f892995..8d5b836 100644 --- a/giftwrap/builders/docker_builder.py +++ b/giftwrap/builders/docker_builder.py @@ -22,7 +22,7 @@ import os import re import tempfile -from giftwrap.builder import Builder +from giftwrap.builders import Builder LOG = logging.getLogger(__name__) @@ -41,7 +41,6 @@ APT_REQUIRED_PACKAGES = [ 'libssl-dev', 'python-dev', 'libmysqlclient-dev', - 'python-virtualenv', 'python-pip', 'build-essential' ] @@ -51,61 +50,71 @@ DEFAULT_SRC_PATH = '/opt/openstack' class DockerBuilder(Builder): def __init__(self, spec): - self.template = DEFAULT_TEMPLATE_FILE self.base_image = 'ubuntu:12.04' self.maintainer = 'maintainer@example.com' self.envvars = {'DEBIAN_FRONTEND': 'noninteractive'} - self._paths = [] + self._commands = [] super(DockerBuilder, self).__init__(spec) - def _validate_settings(self): - pass + def _execute(self, command, cwd=None, exit=0): + if cwd: + self._commands.append("cd %s" % (cwd)) + self._commands.append(command) + if cwd: + self._commands.append("cd -") - def _cleanup(self): - pass + def _make_temp_dir(self, prefix='giftwrap'): + return "/tmp/giftwrap" + self._commands.append("mktemp -d -t %s.XXXXXXXXXX" % prefix) - def _get_prep_commands(self): - commands = [] - commands.append('apt-get update && apt-get install -y %s' % - ' '.join(APT_REQUIRED_PACKAGES)) - return commands + def _make_dir(self, path, mode=0777): + self._commands.append("mkdir -p -m %o %s" % (mode, path)) - def _get_build_commands(self, src_path): - commands = [] - commands.append('mkdir -p %s' % src_path) + def _prepare_project_build(self, project): + return - for project in self._spec.projects: - if project.system_dependencies: - commands.append('apt-get update && apt-get install -y %s' % - ' '.join(project.system_dependencies)) + def _clone_project(self, giturl, name, gitref, depth, path): + cmd = "git clone %s -b %s --depth=%d %s" % (giturl, gitref, + depth, path) + self._commands.append(cmd) - project_src_path = os.path.join(src_path, project.name) - commands.append('git clone --depth 1 %s -b %s %s' % - (project.giturl, project.gitref, project_src_path)) - commands.append('COMMIT=`git rev-parse HEAD` && echo "%s $COMMIT" ' - '> %s/gitinfo' % (project.giturl, - project.install_path)) - commands.append('mkdir -p %s' % - os.path.dirname(project.install_path)) - commands.append('virtualenv --system-site-packages %s' % - project.install_path) + def _create_virtualenv(self, venv_command, path): + self._execute(venv_command, path) - project_bin_path = os.path.join(project.install_path, 'bin') - self._paths.append(project_bin_path) - venv_pip_path = os.path.join(project_bin_path, 'pip') + def _install_pip_dependencies(self, venv_path, dependencies): + pip_path = self._get_venv_pip_path(venv_path) + self._execute("%s install %s" % (pip_path, dependencies)) - if project.pip_dependencies: - commands.append("%s install %s" % (venv_pip_path, - ' '.join(project.pip_dependencies))) - commands.append("%s install %s" % (venv_pip_path, - project_src_path)) + def _copy_sample_config(self, src_clone_dir, project): + src_config = os.path.join(src_clone_dir, 'etc') + dest_config = os.path.join(project.install_path, 'etc') - return commands + self._commands.append("if [ -d %s ]; then cp -R %s %s; fi" % ( + src_config, src_config, dest_config)) - def _get_cleanup_commands(self, src_path): - commands = [] - commands.append('rm -rf %s' % src_path) - return commands + def _install_project(self, venv_path, src_clone_dir): + pip_path = self._get_venv_pip_path(venv_path) + self._execute("%s install %s" % (pip_path, src_clone_dir)) + + def _finalize_project_build(self, project): + self._commands.append("rm -rf %s" % self._temp_dir) + for command in self._commands: + print command + + def _finalize_build(self): + template_vars = { + 'commands': self._commands + } + print self._render_dockerfile(template_vars) + self._build_image() + + def _cleanup_build(self): + return + + def _prepare_build(self): + self._commands.append('apt-get update && apt-get install -y %s' % + ' '.join(APT_REQUIRED_PACKAGES)) + self._commands.append("pip install -U pip virtualenv") def _set_path(self): path = ":".join(self._paths) @@ -116,16 +125,14 @@ class DockerBuilder(Builder): template_vars.update(extra_vars) template_loader = jinja2.FileSystemLoader(searchpath='/') template_env = jinja2.Environment(loader=template_loader) - template = template_env.get_template(self.template) + template = template_env.get_template(DEFAULT_TEMPLATE_FILE) return template.render(template_vars) - def _build(self): - src_path = DEFAULT_SRC_PATH - commands = self._get_prep_commands() - commands += self._get_build_commands(src_path) - commands += self._get_cleanup_commands(src_path) - self._set_path() - dockerfile_contents = self._render_dockerfile(locals()) + def _build_image(self): + template_vars = { + 'commands': self._commands + } + dockerfile_contents = self._render_dockerfile(template_vars) tempdir = tempfile.mkdtemp() dockerfile = os.path.join(tempdir, 'Dockerfile') diff --git a/giftwrap/builders/package_builder.py b/giftwrap/builders/package_builder.py index 530808e..b25c1c1 100644 --- a/giftwrap/builders/package_builder.py +++ b/giftwrap/builders/package_builder.py @@ -20,8 +20,7 @@ import os import shutil import tempfile -from giftwrap.builder import Builder -from giftwrap.gerrit import GerritReview +from giftwrap.builders import Builder from giftwrap.openstack_git_repo import OpenstackGitRepo from giftwrap.package import Package from giftwrap.util import execute @@ -32,95 +31,73 @@ LOG = logging.getLogger(__name__) class PackageBuilder(Builder): def __init__(self, spec): - self._tempdir = None + self._temp_dir = None super(PackageBuilder, self).__init__(spec) - def _validate_settings(self): - pass + def _execute(self, command, cwd=None, exit=0): + return execute(command, cwd, exit) - def _install_gerrit_dependencies(self, repo, project, install_path): - try: - review = GerritReview(repo.head.change_id, project.git_path) - LOG.info("Installing '%s' pip dependencies to the virtualenv", - project.name) - execute(project.install_command % - review.build_pip_dependencies(string=True), install_path) - except Exception as e: - LOG.warning("Could not install gerrit dependencies!!! " - "Error was: %s", e) + def _make_temp_dir(self, prefix='giftwrap'): + return tempfile.mkdtemp(prefix) - def _build(self): - spec = self._spec + def _make_dir(self, path, mode=0777): + os.makedirs(path, mode) - self._tempdir = tempfile.mkdtemp(prefix='giftwrap') - src_path = os.path.join(self._tempdir, 'src') - LOG.debug("Temporary working directory: %s", self._tempdir) + def _prepare_build(self): + return - for project in spec.projects: - LOG.info("Beginning to build '%s'", project.name) + def _prepare_project_build(self, project): + install_path = project.install_path - install_path = project.install_path - LOG.debug("Installing '%s' to '%s'", project.name, install_path) + LOG.info("Beginning to build '%s'", project.name) + if os.path.exists(install_path): + if self._spec.settings.force_overwrite: + LOG.info("force_overwrite is set, so removing " + "existing path '%s'" % install_path) + shutil.rmtree(install_path) + else: + raise Exception("Install path '%s' already exists" % + install_path) - # if anything is in our way, see if we can get rid of it - if os.path.exists(install_path): - if spec.settings.force_overwrite: - LOG.info("force_overwrite is set, so removing " - "existing path '%s'" % install_path) - shutil.rmtree(install_path) - else: - raise Exception("Install path '%s' already exists" % - install_path) - os.makedirs(install_path) + def _clone_project(self, giturl, name, gitref, depth, path): + LOG.info("Fetching source code for '%s'", name) + repo = OpenstackGitRepo(giturl, name, gitref, depth) + repo.clone(path) + return repo - # clone the project's source to a temporary directory - project_src_path = os.path.join(src_path, project.name) - os.makedirs(project_src_path) + def _create_virtualenv(self, venv_command, path): + self._execute(venv_command, path) - LOG.info("Fetching source code for '%s'", project.name) - repo = OpenstackGitRepo(project.giturl, project.name, - project.gitref, - depth=project.gitdepth) - repo.clone(project_src_path) + def _install_pip_dependencies(self, venv_path, dependencies): + pip_path = self._get_venv_pip_path(venv_path) + self._execute("%s install %s" % (pip_path, dependencies)) - # tell package users where this came from - gitinfo_file = os.path.join(install_path, 'gitinfo') - with open(gitinfo_file, 'w') as fh: - fh.write("%s %s" % (project.giturl, repo.head.hexsha)) + def _copy_sample_config(self, src_clone_dir, project): + src_config = os.path.join(src_clone_dir, 'etc') + dest_config = os.path.join(project.install_path, 'etc') - # start building the virtualenv for the project - LOG.info("Creating the virtualenv for '%s'", project.name) - execute(project.venv_command, install_path) + if not os.path.exists(src_config): + LOG.warning("Project configuration does not seem to exist " + "in source repo '%s'. Skipping.", project.name) + else: + LOG.debug("Copying config from '%s' to '%s'", src_config, + dest_config) + distutils.dir_util.copy_tree(src_config, dest_config) - # install into the virtualenv - LOG.info("Installing '%s' to the virtualenv", project.name) - venv_pip_path = os.path.join(install_path, 'bin/pip') + def _install_project(self, venv_path, src_clone_dir): + pip_path = self._get_venv_pip_path(venv_path) + self._execute("%s install %s" % (pip_path, src_clone_dir)) - deps = " ".join(project.pip_dependencies) - execute("%s install %s" % (venv_pip_path, deps)) + def _finalize_project_build(self, project): + # build the package + pkg = Package(project.package_name, project.version, + project.install_path, self._spec.settings.output_dir, + self._spec.settings.force_overwrite, + project.system_dependencies) + pkg.build() - if spec.settings.include_config: - src_config = os.path.join(project_src_path, 'etc') - dest_config = os.path.join(install_path, 'etc') - if not os.path.exists(src_config): - LOG.warning("Project configuration does not seem to exist " - "in source repo '%s'. Skipping.", project.name) - else: - LOG.debug("Copying config from '%s' to '%s'", src_config, - dest_config) - distutils.dir_util.copy_tree(src_config, dest_config) + def _finalize_build(self): + return - if spec.settings.gerrit_dependencies: - self._install_gerrit_dependencies(repo, project, install_path) - - execute("%s install %s" % (venv_pip_path, project_src_path)) - - # now build the package - pkg = Package(project.package_name, project.version, - install_path, spec.settings.output_dir, - spec.settings.force_overwrite, - project.system_dependencies) - pkg.build() - - def _cleanup(self): - shutil.rmtree(self._tempdir) + def _cleanup_build(self): + shutil.rmtree(self._temp_dir) diff --git a/giftwrap/gerrit.py b/giftwrap/gerrit.py index 042989e..f9309c5 100644 --- a/giftwrap/gerrit.py +++ b/giftwrap/gerrit.py @@ -50,7 +50,8 @@ class GerritReview(object): freeze_found = True continue elif re.match('[\w\-]+==.+', line) and not line.startswith('-e'): - dependencies.append(line) + dependency = line.split('#')[0].strip() # remove any comments + dependencies.append(dependency) short_name = self.project.split('/')[1] dependencies = filter(lambda x: not x.startswith(short_name + "=="), diff --git a/giftwrap/openstack_project.py b/giftwrap/openstack_project.py index 3a41868..ff7ef38 100644 --- a/giftwrap/openstack_project.py +++ b/giftwrap/openstack_project.py @@ -23,7 +23,7 @@ DEFAULT_GITURL = { 'openstack': 'https://git.openstack.org/openstack/', 'stackforge': 'https://github.com/stackforge/' } -DEFAULT_VENV_COMMAND = "virtualenv ." +DEFAULT_VENV_COMMAND = "virtualenv --no-wheel ." DEFAULT_INSTALL_COMMAND = "./bin/pip install %s" # noqa TEMPLATE_VARS = ('name', 'version', 'gitref', 'stackforge') @@ -32,7 +32,7 @@ TEMPLATE_VARS = ('name', 'version', 'gitref', 'stackforge') class OpenstackProject(object): def __init__(self, settings, name, version=None, gitref=None, giturl=None, - gitdepth=None, venv_command=None, install_command=None, + gitdepth=1, venv_command=None, install_command=None, install_path=None, package_name=None, stackforge=False, system_dependencies=[], pip_dependencies=[]): self._settings = settings diff --git a/giftwrap/shell.py b/giftwrap/shell.py index bd2e89f..a461fed 100644 --- a/giftwrap/shell.py +++ b/giftwrap/shell.py @@ -19,8 +19,7 @@ import logging import signal import sys -import giftwrap.builder - +from giftwrap.builders import BuilderFactory from giftwrap.build_spec import BuildSpec from giftwrap.color import ColorStreamHandler @@ -48,7 +47,7 @@ def build(args): manifest = fh.read() buildspec = BuildSpec(manifest, args.version, args.type) - builder = giftwrap.builder.create_builder(buildspec) + builder = BuilderFactory.create_builder(args.type, buildspec) def _signal_handler(*args): LOG.info("Process interrrupted. Cleaning up.") @@ -79,7 +78,8 @@ def main(): description='build giftwrap packages') build_subcmd.add_argument('-m', '--manifest', required=True) build_subcmd.add_argument('-v', '--version') - build_subcmd.add_argument('-t', '--type', choices=('docker', 'package')) + build_subcmd.add_argument('-t', '--type', choices=('docker', 'package'), + required=True) build_subcmd.set_defaults(func=build) args = parser.parse_args() diff --git a/giftwrap/templates/Dockerfile.jinja2 b/giftwrap/templates/Dockerfile.jinja2 index bb90490..13374c6 100644 --- a/giftwrap/templates/Dockerfile.jinja2 +++ b/giftwrap/templates/Dockerfile.jinja2 @@ -7,6 +7,4 @@ MAINTAINER {{ maintainer }} ENV {{ k }} {{ v }} {% endfor -%} -{% for command in commands -%} -RUN {{ command }} -{% endfor %} +RUN {% for command in commands[:-1] -%}{{ command|safe }} && {% endfor -%} {{ commands[-1]|safe }}