From 3f04bd80718461d2907f0d4b4be140143121ee6c Mon Sep 17 00:00:00 2001
From: Isaac Beckman <isaacb@il.ibm.com>
Date: Sun, 6 Nov 2016 11:35:11 +0200
Subject: [PATCH] cleanup: remove unused scripts used for snapshots
This review cleanup scripts that were exclusively used by
snapshot scripts for nodepool which were removed in:
https://review.openstack.org/#/c/331494/
Change-Id: I37900a5d698a43bc0d510100289a5a7ec527021d
---
nodepool/scripts/cache_devstack.py | 189 ------------------
nodepool/scripts/cache_git_repos.py | 92 ---------
nodepool/scripts/common.py | 32 ---
nodepool/scripts/fix_grub_timeout.sh | 31 ---
.../scripts/install_devstack_dependencies.sh | 52 -----
5 files changed, 396 deletions(-)
delete mode 100755 nodepool/scripts/cache_devstack.py
delete mode 100755 nodepool/scripts/cache_git_repos.py
delete mode 100644 nodepool/scripts/common.py
delete mode 100755 nodepool/scripts/fix_grub_timeout.sh
delete mode 100755 nodepool/scripts/install_devstack_dependencies.sh
diff --git a/nodepool/scripts/cache_devstack.py b/nodepool/scripts/cache_devstack.py
deleted file mode 100755
index 0c7505959b..0000000000
--- a/nodepool/scripts/cache_devstack.py
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2011-2013 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-
-from common import run_local
-
-DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
-CACHEDIR = os.path.expanduser('~/cache/files')
-
-# Some jobs might require newer distro packages, so we can pre-cache
-# deb packages from specified Ubuntu Cloud Archive pockets.
-UCA_POCKETS = []
-
-
-def git_branches():
- branches = []
- for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
- branch = branch.strip()
- if not branch.startswith('remotes/origin'):
- continue
- branches.append(branch)
- return branches
-
-
-def tokenize(fn, tokens, distribution, comment=None):
- for line in open(fn):
- if 'dist:' in line and ('dist:%s' % distribution not in line):
- continue
- if 'qpid' in line:
- continue # TODO: explain why this is here
- if comment and comment in line:
- line = line[:line.find(comment)]
- line = line.strip()
- if line and line not in tokens:
- tokens.append(line)
-
-
-def _legacy_find_images(basedir):
- """Divine what images we should use based on parsing stackrc."""
- images = []
- for line in open(os.path.join(basedir, 'stackrc')):
- line = line.strip()
- if line.startswith('IMAGE_URLS'):
- if '#' in line:
- line = line[:line.find('#')]
- if line.endswith(';;'):
- line = line[:-2]
- line = line.split('=', 1)[1].strip()
- if line.startswith('${IMAGE_URLS:-'):
- line = line[len('${IMAGE_URLS:-'):]
- if line.endswith('}'):
- line = line[:-1]
- if not line:
- continue
- if line[0] == line[-1] == '"':
- line = line[1:-1]
- # Add image to the list to be downloaded, but
- # skip downloading giant vmware images
- images += [x.strip() for x in line.split(',')
- if not x.strip().endswith('vmdk')]
- return images
-
-
-def _find_images(basedir):
- images = []
- image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
- if os.path.exists(image_tool):
- returncode, out = run_local(image_tool, status=True)
- if returncode:
- print "%s failed" % image_tool
- print "Exit: %s, Output: %s" % (returncode, out)
- # reset images so we'll fall back
- images = []
- else:
- images = out.split('\n')
- return images
-
-
-def local_prep(distribution):
- branches = []
- for branch in git_branches():
- # Ignore branches of the form 'somestring -> someotherstring'
- # as this denotes a symbolic reference and the entire string
- # as is cannot be checked out. We can do this safely as the
- # reference will refer to one of the other branches returned
- # by git_branches.
- if ' -> ' in branch:
- continue
- branch_data = {'name': branch}
- print 'Branch: ', branch
- run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
- run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
-
- if os.path.exists('/usr/bin/apt-get'):
- debs = []
- debdir = os.path.join(DEVSTACK, 'files', 'debs')
- if not os.path.exists(debdir):
- debdir = os.path.join(DEVSTACK, 'files', 'apts')
- for fn in os.listdir(debdir):
- fn = os.path.join(debdir, fn)
- tokenize(fn, debs, distribution, comment='#')
- branch_data['debs'] = debs
-
- if os.path.exists('/usr/bin/yum'):
- rpms = []
- rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
- for fn in os.listdir(rpmdir):
- fn = os.path.join(rpmdir, fn)
- tokenize(fn, rpms, distribution, comment='#')
- branch_data['rpms'] = rpms
-
- images = _find_images(DEVSTACK)
- if not images:
- images = _legacy_find_images(DEVSTACK)
-
- branch_data['images'] = images
- branches.append(branch_data)
- return branches
-
-
-def download(url, fname):
- run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
-
-
-def cache_debs(debs, uca_pocket=None):
- """Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
- Archive pocket. If a UCA pocket is specified, it is enabled temporarily
- for caching only.
- """
- if uca_pocket:
- # Note this will install the ubuntu-cloud-keyring package which
- # contains the required GPG key.
- run_local(['sudo', 'add-apt-repository', '-y',
- 'cloud-archive:%s' % uca_pocket])
- run_local(['sudo', 'apt-get', 'update'])
- run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
- if uca_pocket:
- run_local(['sudo', 'rm', '-f',
- '/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
- run_local(['sudo', 'apt-get', 'update'])
-
-
-def main():
- distribution = sys.argv[1]
-
- branches = local_prep(distribution)
- image_filenames = []
- for branch_data in branches:
- if branch_data.get('debs'):
- cache_debs(branch_data['debs'])
- for uca in sorted(UCA_POCKETS):
- cache_debs(branch_data['debs'], uca)
- elif branch_data.get('rpms'):
- run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
- branch_data['rpms'])
- else:
- sys.exit('No supported package data found.')
-
- for url in branch_data['images']:
- fname = url.split('/')[-1]
- if fname in image_filenames:
- continue
- image_filenames.append(fname)
- download(url, fname)
-
- # cache get-pip, because upstream network connection fails more
- # often than you might imagine.
- download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
-
-
-if __name__ == '__main__':
- main()
diff --git a/nodepool/scripts/cache_git_repos.py b/nodepool/scripts/cache_git_repos.py
deleted file mode 100755
index 46b5758273..0000000000
--- a/nodepool/scripts/cache_git_repos.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2011-2013 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os.path
-import re
-import shutil
-import sys
-import urllib2
-
-from common import run_local
-
-URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
- 'plain/gerrit/projects.yaml')
-PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$')
-
-# Not using an arg libraries in order to avoid module imports that
-# are not available across all python versions
-if len(sys.argv) > 1:
- GIT_BASE = sys.argv[1]
-else:
- GIT_BASE = 'git://git.openstack.org'
-
-
-def clone_repo(project):
- remote = '%s/%s.git' % (GIT_BASE, project)
-
- # Clear out any existing target directory first, in case of a retry.
- try:
- shutil.rmtree(os.path.join('/opt/git', project))
- except OSError:
- pass
-
- # Try to clone the requested git repository.
- (status, out) = run_local(['git', 'clone', remote, project],
- status=True, cwd='/opt/git')
-
- # If it claims to have worked, make sure we can list branches.
- if status == 0:
- (status, moreout) = run_local(['git', 'branch', '-a'], status=True,
- cwd=os.path.join('/opt/git', project))
- out = '\n'.join((out, moreout))
-
- # If that worked, try resetting to HEAD to make sure it's there.
- if status == 0:
- (status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
- status=True,
- cwd=os.path.join('/opt/git', project))
- out = '\n'.join((out, moreout))
-
- # Status of 0 imples all the above worked, 1 means something failed.
- return (status, out)
-
-
-def main():
- # TODO(jeblair): use gerrit rest api when available
- data = urllib2.urlopen(URL).read()
- for line in data.split('\n'):
- # We're regex-parsing YAML so that we don't have to depend on the
- # YAML module which is not in the stdlib.
- m = PROJECT_RE.match(line)
- if m:
- project = m.group(1)
- dirname = os.path.dirname(project)
- # Skip repos that are inactive
- if not ('attic' in dirname or dirname == 'stackforge'):
- (status, out) = clone_repo(project)
- print out
- if status != 0:
- print 'Retrying to clone %s' % m.group(1)
- (status, out) = clone_repo(m.group(1))
- print out
- if status != 0:
- raise Exception('Failed to clone %s' % m.group(1))
-
-
-if __name__ == '__main__':
- main()
diff --git a/nodepool/scripts/common.py b/nodepool/scripts/common.py
deleted file mode 100644
index 3ccfedfdf2..0000000000
--- a/nodepool/scripts/common.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2011-2013 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import subprocess
-
-
-def run_local(cmd, status=False, cwd='.', env={}):
- print "Running:", cmd
- newenv = os.environ
- newenv.update(env)
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
- stderr=subprocess.STDOUT, env=newenv)
- (out, nothing) = p.communicate()
- if status:
- return (p.returncode, out.strip())
- return out.strip()
diff --git a/nodepool/scripts/fix_grub_timeout.sh b/nodepool/scripts/fix_grub_timeout.sh
deleted file mode 100755
index 77fb04b218..0000000000
--- a/nodepool/scripts/fix_grub_timeout.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash -xe
-# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Set the grub timeout to 0.
-if [ -f /etc/default/grub ] ; then
- sudo sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' \
- /etc/default/grub
- if which update-grub &> /dev/null ; then
- sudo update-grub
- else
- # If update-grub isn't available, use grub2-mkconfig directly
- sudo /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
- fi
-elif [ -f /boot/grub/grub.conf ] ; then
- sudo sed -i -e 's/^timeout=[0-9]\+/timeout=0/' \
- /boot/grub/grub.conf
-fi
diff --git a/nodepool/scripts/install_devstack_dependencies.sh b/nodepool/scripts/install_devstack_dependencies.sh
deleted file mode 100755
index a1c713e037..0000000000
--- a/nodepool/scripts/install_devstack_dependencies.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash -xe
-
-# Copyright (C) 2011-2013 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-mkdir -p ~/cache/files
-mkdir -p ~/cache/pip
-
-if [ -f /usr/bin/yum ]; then
- sudo yum -y install python-devel python3-devel make automake gcc gcc-c++ \
- kernel-devel redhat-lsb-core
-elif [ -f /usr/bin/apt-get ]; then
- if [ "$(lsb_release -c -s)" = "precise" ]; then
- # temporary - remove the breaking -70 kernel and
- # reinstall -69 to prevent openvswitch breakage
- sudo DEBIAN_FRONTEND=noninteractive apt-get \
- --option "Dpkg::Options::=--force-confold" \
- --assume-yes remove linux-headers-3.2.0-70 \
- linux-headers-3.2.0-70-virtual \
- linux-image-3.2.0-70-virtual \
- linux-headers-3.2.0-70-generic \
- linux-image-3.2.0-70-generic
- sudo DEBIAN_FRONTEND=noninteractive apt-get \
- --option "Dpkg::Options::=--force-confold" \
- --assume-yes install --reinstall linux-headers-3.2.0-69 \
- linux-headers-3.2.0-69-virtual \
- linux-image-3.2.0-69-virtual \
- python-software-properties build-essential python-dev python3-dev
- else
- sudo DEBIAN_FRONTEND=noninteractive apt-get \
- --option "Dpkg::Options::=--force-confold" \
- --assume-yes install build-essential python-dev python3-dev \
- python-software-properties linux-headers-virtual \
- linux-headers-$(uname -r)
- fi
-else
- echo "Unsupported distro."
- exit 1
-fi