From f0bec296c63c24beeb9b01c849be2523fcb70be7 Mon Sep 17 00:00:00 2001 From: Bhuvan Arumugam Date: Wed, 5 Feb 2014 22:54:13 +0530 Subject: [PATCH] Load project settings from ini file, if present. We manage project settings in 2 files, projects.ini and projects.yaml file. If ini file is present, read defaults from this file and parse yaml using new format. Closes-Bug: 1275824 Change-Id: I5bc2e81388420ce03d5379b8d75063481887243b --- jeepyb/cmd/close_pull_requests.py | 15 ++--- jeepyb/cmd/create_cgitrepos.py | 17 +++--- jeepyb/cmd/manage_projects.py | 96 ++++++++----------------------- jeepyb/projects.py | 3 +- jeepyb/utils.py | 48 ++++++++++++---- 5 files changed, 73 insertions(+), 106 deletions(-) diff --git a/jeepyb/cmd/close_pull_requests.py b/jeepyb/cmd/close_pull_requests.py index 19d2d25..f890bb1 100644 --- a/jeepyb/cmd/close_pull_requests.py +++ b/jeepyb/cmd/close_pull_requests.py @@ -42,7 +42,8 @@ import ConfigParser import github import logging import os -import yaml + +import jeepyb.utils as u MESSAGE = """Thank you for contributing to %(project)s! @@ -57,20 +58,12 @@ def main(): logging.basicConfig(level=logging.ERROR) - PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') - PROJECTS_INI = os.environ.get('PROJECTS_INI', - '/home/gerrit2/projects.ini') GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', '/etc/github/github.secure.config') secure_config = ConfigParser.ConfigParser() secure_config.read(GITHUB_SECURE_CONFIG) - yaml_docs = [config for config in yaml.load_all(open(PROJECTS_YAML))] - if os.path.exists(PROJECTS_INI): - config = yaml_docs[0] - else: - config = yaml_docs[1] + registry = u.ProjectsRegistry() if secure_config.has_option("github", "oauth_token"): ghub = github.Github(secure_config.get("github", "oauth_token")) @@ -80,7 +73,7 @@ def main(): orgs = ghub.get_user().get_orgs() orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) - for section in config: + for section in registry.configs_list: project = section['project'] # Make sure we're supposed to close pull requests for this project: diff --git a/jeepyb/cmd/create_cgitrepos.py b/jeepyb/cmd/create_cgitrepos.py index 1572a44..6950655 100644 --- a/jeepyb/cmd/create_cgitrepos.py +++ b/jeepyb/cmd/create_cgitrepos.py @@ -22,15 +22,13 @@ import os import subprocess -import yaml + +import jeepyb.utils as u -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/cgit/projects.yaml') -CGIT_REPOS = os.environ.get('CGIT_REPOS', - '/etc/cgitrepos') -REPO_PATH = os.environ.get('REPO_PATH', - '/var/lib/git') +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', '/home/cgit/projects.yaml') +CGIT_REPOS = os.environ.get('CGIT_REPOS', '/etc/cgitrepos') +REPO_PATH = os.environ.get('REPO_PATH', '/var/lib/git') SCRATCH_SUBPATH = os.environ.get('SCRATCH_SUBPATH') SCRATCH_OWNER = os.environ.get('SCRATCH_OWNER', 'scratch') SCRATCH_GROUP = os.environ.get('SCRATCH_GROUP', 'scratch') @@ -39,11 +37,10 @@ CGIT_GROUP = os.environ.get('CGIT_GROUP', 'cgit') def main(): - yaml_docs = [config for config in yaml.safe_load_all(open(PROJECTS_YAML))] - config = yaml_docs[-1] + registry = u.ProjectsRegistry(PROJECTS_YAML) gitorgs = {} names = set() - for entry in config: + for entry in registry.configs_list: project = entry['project'] (org, name) = project.split('/') description = entry.get('description', name) diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py index c446244..3ee4cd2 100644 --- a/jeepyb/cmd/manage_projects.py +++ b/jeepyb/cmd/manage_projects.py @@ -58,12 +58,14 @@ import shlex import subprocess import tempfile import time -import yaml import gerritlib.gerrit import github import jeepyb.gerritdb +import jeepyb.utils as u + +registry = u.ProjectsRegistry() log = logging.getLogger("manage_projects") @@ -505,15 +507,6 @@ def create_local_mirror(local_git_dir, project_git, git_mirror_path)) -def get_option(options, section, key, default): - if options.has_option(section, key): - if type(default) is bool: - return options.getboolean(section, key) - else: - return options.get(section, key) - return default - - def main(): parser = argparse.ArgumentParser(description='Manage projects') parser.add_argument('-v', dest='verbose', action='store_true', @@ -533,68 +526,27 @@ def main(): else: logging.basicConfig(level=logging.ERROR) - PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') - yaml_docs = [config for config in yaml.safe_load_all(open(PROJECTS_YAML))] + default_has_github = registry.get('has-github', True) - PROJECTS_INI = os.environ.get('PROJECTS_INI', - '/home/gerrit2/projects.ini') - if os.path.exists(PROJECTS_INI): - # New-style - supports projects.ini - projects_yaml_list = yaml_docs[0] - defaults = ConfigParser.ConfigParser() - defaults.read(PROJECTS_INI) - - default_has_github = get_option( - defaults, 'projects', 'has-github', True) - - LOCAL_GIT_DIR = get_option( - defaults, 'projects', 'local-git-dir', '/var/lib/git') - JEEPYB_CACHE_DIR = get_option( - defaults, 'projects', 'jeepyb-cache-dir', '/var/lib/jeepyb') - ACL_DIR = defaults.get('projects', 'acl-dir') - GERRIT_HOST = defaults.get('projects', 'gerrit-host') - GERRIT_PORT = int(get_option( - defaults, 'projects', 'gerrit-port', '29418')) - GERRIT_USER = defaults.get('projects', 'gerrit-user') - GERRIT_KEY = defaults.get('projects', 'gerrit-key') - GERRIT_GITID = defaults.get('projects', 'gerrit-committer') - GERRIT_SYSTEM_USER = get_option( - defaults, 'projects', 'gerrit-system-user', 'gerrit2') - GERRIT_SYSTEM_GROUP = get_option( - defaults, 'projects', 'gerrit-system-group', 'gerrit2') - DEFAULT_HOMEPAGE = get_option(defaults, 'projects', 'homepage', None) - DEFAULT_HAS_ISSUES = get_option( - defaults, 'projects', 'has-issues', False) - DEFAULT_HAS_DOWNLOADS = get_option( - defaults, 'projects', 'has-downloads', False) - DEFAULT_HAS_WIKI = get_option(defaults, 'projects', 'has-wiki', False) - GITHUB_SECURE_CONFIG = get_option( - defaults, 'projects', 'github-config', - '/etc/github/github-projects.secure.config') - else: - # Old-style - embedded - projects_yaml_list = yaml_docs[1] - defaults = yaml_docs[0][0] - default_has_github = defaults.get('has-github', True) - - LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git') - JEEPYB_CACHE_DIR = defaults.get('jeepyb-cache-dir', '/var/lib/jeepyb') - ACL_DIR = defaults.get('acl-dir') - GERRIT_HOST = defaults.get('gerrit-host') - GERRIT_PORT = int(defaults.get('gerrit-port', '29418')) - GERRIT_USER = defaults.get('gerrit-user') - GERRIT_KEY = defaults.get('gerrit-key') - GERRIT_GITID = defaults.get('gerrit-committer') - GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2') - GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2') - DEFAULT_HOMEPAGE = defaults.get('homepage', None) - DEFAULT_HAS_ISSUES = defaults.get('has-issues', False) - DEFAULT_HAS_DOWNLOADS = defaults.get('has-downloads', False) - DEFAULT_HAS_WIKI = defaults.get('has-wiki', False) - GITHUB_SECURE_CONFIG = defaults.get( - 'github-config', - '/etc/github/github-projects.secure.config') + LOCAL_GIT_DIR = registry.get_defaults('local-git-dir', '/var/lib/git') + JEEPYB_CACHE_DIR = registry.get_defaults('jeepyb-cache-dir', + '/var/lib/jeepyb') + ACL_DIR = registry.get_defaults('acl-dir') + GERRIT_HOST = registry.get_defaults('gerrit-host') + GERRIT_PORT = int(registry.get_defaults('gerrit-port', '29418')) + GERRIT_USER = registry.get_defaults('gerrit-user') + GERRIT_KEY = registry.get_defaults('gerrit-key') + GERRIT_GITID = registry.get_defaults('gerrit-committer') + GERRIT_SYSTEM_USER = registry.get_defaults('gerrit-system-user', 'gerrit2') + GERRIT_SYSTEM_GROUP = registry.get_defaults('gerrit-system-group', + 'gerrit2') + DEFAULT_HOMEPAGE = registry.get_defaults('homepage') + DEFAULT_HAS_ISSUES = registry.get_defaults('has-issues', False) + DEFAULT_HAS_DOWNLOADS = registry.get_defaults('has-downloads', False) + DEFAULT_HAS_WIKI = registry.get_defaults('has-wiki', False) + GITHUB_SECURE_CONFIG = registry.get_defaults( + 'github-config', + '/etc/github/github-projects.secure.config') gerrit = gerritlib.gerrit.Gerrit('localhost', GERRIT_USER, @@ -604,7 +556,7 @@ def main(): ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) try: - for section in projects_yaml_list: + for section in registry.configs_list: project = section['project'] if args.projects and project not in args.projects: continue diff --git a/jeepyb/projects.py b/jeepyb/projects.py index 64d00d9..a801830 100644 --- a/jeepyb/projects.py +++ b/jeepyb/projects.py @@ -27,8 +27,7 @@ Expected review.projects.yaml format: import jeepyb.utils as u -registry = u.ProjectsYamlRegistry('/home/gerrit2/projects.yaml', - 'PROJECTS_YAML') +registry = u.ProjectsRegistry() def project_to_group(project_full_name): diff --git a/jeepyb/utils.py b/jeepyb/utils.py index f92852e..7b04dc1 100644 --- a/jeepyb/utils.py +++ b/jeepyb/utils.py @@ -12,38 +12,49 @@ # License for the specific language governing permissions and limitations # under the License. +import ConfigParser import os import yaml +PROJECTS_INI = os.environ.get('PROJECTS_INI', '/home/gerrit2/projects.ini') +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', '/home/gerrit2/projects.yaml') + def short_project_name(full_project_name): """Return the project part of the git repository name.""" return full_project_name.split('/')[-1] -class ProjectsYamlRegistry(object): - """review.projects.yaml style config file parser. +class ProjectsRegistry(object): + """read config from ini or yaml file. It could be used as dict 'project name' -> 'project properties'. """ - - def __init__(self, file_path, env_name=None, single_doc=True): - self.file_path = file_path - self.env_name = env_name + def __init__(self, yaml_file=PROJECTS_YAML, single_doc=True): + self.yaml_doc = [c for c in yaml.safe_load_all(open(yaml_file))] self.single_doc = single_doc + self.configs_list = [] + self.defaults = {} self._parse_file() def _parse_file(self): - file_path = os.environ.get(self.env_name, self.file_path) - yaml_docs = [config for config in yaml.safe_load_all(open(file_path))] if self.single_doc: - configs_list = yaml_docs[0] + self.configs_list = self.yaml_doc[0] else: - configs_list = yaml_docs[1] + self.configs_list = self.yaml_doc[1] + + if os.path.exists(PROJECTS_INI): + self.defaults = ConfigParser.ConfigParser() + self.defaults.read(PROJECTS_INI) + else: + try: + self.defaults = self.yaml_doc[0][0] + except IndexError: + pass configs = {} - for section in configs_list: + for section in self.configs_list: configs[section['project']] = section self.configs = configs @@ -56,3 +67,18 @@ class ProjectsYamlRegistry(object): return self.configs[project].get(item, default) else: return default + + def get(self, item, default=None): + return self.configs.get(item, default) + + def get_defaults(self, item, default=None): + if os.path.exists(PROJECTS_INI): + section = 'projects' + if self.defaults.has_option(section, item): + if type(default) == bool: + return self.defaults.getboolean(section, item) + else: + return self.defaults.get(section, item) + return default + else: + return self.defaults.get(item, default)