cleanup: remove unused scripts used for snapshots
This review cleanup scripts that were exclusively used by snapshot scripts for nodepool which were removed in: https://review.openstack.org/#/c/331494/ Change-Id: I37900a5d698a43bc0d510100289a5a7ec527021d
This commit is contained in:
Isaac Beckman
@@ -1,189 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
#
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from common import run_local
|
|
||||||
|
|
||||||
DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
|
|
||||||
CACHEDIR = os.path.expanduser('~/cache/files')
|
|
||||||
|
|
||||||
# Some jobs might require newer distro packages, so we can pre-cache
|
|
||||||
# deb packages from specified Ubuntu Cloud Archive pockets.
|
|
||||||
UCA_POCKETS = []
|
|
||||||
|
|
||||||
|
|
||||||
def git_branches():
|
|
||||||
branches = []
|
|
||||||
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
|
|
||||||
branch = branch.strip()
|
|
||||||
if not branch.startswith('remotes/origin'):
|
|
||||||
continue
|
|
||||||
branches.append(branch)
|
|
||||||
return branches
|
|
||||||
|
|
||||||
|
|
||||||
def tokenize(fn, tokens, distribution, comment=None):
|
|
||||||
for line in open(fn):
|
|
||||||
if 'dist:' in line and ('dist:%s' % distribution not in line):
|
|
||||||
continue
|
|
||||||
if 'qpid' in line:
|
|
||||||
continue # TODO: explain why this is here
|
|
||||||
if comment and comment in line:
|
|
||||||
line = line[:line.find(comment)]
|
|
||||||
line = line.strip()
|
|
||||||
if line and line not in tokens:
|
|
||||||
tokens.append(line)
|
|
||||||
|
|
||||||
|
|
||||||
def _legacy_find_images(basedir):
|
|
||||||
"""Divine what images we should use based on parsing stackrc."""
|
|
||||||
images = []
|
|
||||||
for line in open(os.path.join(basedir, 'stackrc')):
|
|
||||||
line = line.strip()
|
|
||||||
if line.startswith('IMAGE_URLS'):
|
|
||||||
if '#' in line:
|
|
||||||
line = line[:line.find('#')]
|
|
||||||
if line.endswith(';;'):
|
|
||||||
line = line[:-2]
|
|
||||||
line = line.split('=', 1)[1].strip()
|
|
||||||
if line.startswith('${IMAGE_URLS:-'):
|
|
||||||
line = line[len('${IMAGE_URLS:-'):]
|
|
||||||
if line.endswith('}'):
|
|
||||||
line = line[:-1]
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
if line[0] == line[-1] == '"':
|
|
||||||
line = line[1:-1]
|
|
||||||
# Add image to the list to be downloaded, but
|
|
||||||
# skip downloading giant vmware images
|
|
||||||
images += [x.strip() for x in line.split(',')
|
|
||||||
if not x.strip().endswith('vmdk')]
|
|
||||||
return images
|
|
||||||
|
|
||||||
|
|
||||||
def _find_images(basedir):
|
|
||||||
images = []
|
|
||||||
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
|
|
||||||
if os.path.exists(image_tool):
|
|
||||||
returncode, out = run_local(image_tool, status=True)
|
|
||||||
if returncode:
|
|
||||||
print "%s failed" % image_tool
|
|
||||||
print "Exit: %s, Output: %s" % (returncode, out)
|
|
||||||
# reset images so we'll fall back
|
|
||||||
images = []
|
|
||||||
else:
|
|
||||||
images = out.split('\n')
|
|
||||||
return images
|
|
||||||
|
|
||||||
|
|
||||||
def local_prep(distribution):
|
|
||||||
branches = []
|
|
||||||
for branch in git_branches():
|
|
||||||
# Ignore branches of the form 'somestring -> someotherstring'
|
|
||||||
# as this denotes a symbolic reference and the entire string
|
|
||||||
# as is cannot be checked out. We can do this safely as the
|
|
||||||
# reference will refer to one of the other branches returned
|
|
||||||
# by git_branches.
|
|
||||||
if ' -> ' in branch:
|
|
||||||
continue
|
|
||||||
branch_data = {'name': branch}
|
|
||||||
print 'Branch: ', branch
|
|
||||||
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
|
|
||||||
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
|
|
||||||
|
|
||||||
if os.path.exists('/usr/bin/apt-get'):
|
|
||||||
debs = []
|
|
||||||
debdir = os.path.join(DEVSTACK, 'files', 'debs')
|
|
||||||
if not os.path.exists(debdir):
|
|
||||||
debdir = os.path.join(DEVSTACK, 'files', 'apts')
|
|
||||||
for fn in os.listdir(debdir):
|
|
||||||
fn = os.path.join(debdir, fn)
|
|
||||||
tokenize(fn, debs, distribution, comment='#')
|
|
||||||
branch_data['debs'] = debs
|
|
||||||
|
|
||||||
if os.path.exists('/usr/bin/yum'):
|
|
||||||
rpms = []
|
|
||||||
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
|
|
||||||
for fn in os.listdir(rpmdir):
|
|
||||||
fn = os.path.join(rpmdir, fn)
|
|
||||||
tokenize(fn, rpms, distribution, comment='#')
|
|
||||||
branch_data['rpms'] = rpms
|
|
||||||
|
|
||||||
images = _find_images(DEVSTACK)
|
|
||||||
if not images:
|
|
||||||
images = _legacy_find_images(DEVSTACK)
|
|
||||||
|
|
||||||
branch_data['images'] = images
|
|
||||||
branches.append(branch_data)
|
|
||||||
return branches
|
|
||||||
|
|
||||||
|
|
||||||
def download(url, fname):
|
|
||||||
run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
|
|
||||||
|
|
||||||
|
|
||||||
def cache_debs(debs, uca_pocket=None):
|
|
||||||
"""Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
|
|
||||||
Archive pocket. If a UCA pocket is specified, it is enabled temporarily
|
|
||||||
for caching only.
|
|
||||||
"""
|
|
||||||
if uca_pocket:
|
|
||||||
# Note this will install the ubuntu-cloud-keyring package which
|
|
||||||
# contains the required GPG key.
|
|
||||||
run_local(['sudo', 'add-apt-repository', '-y',
|
|
||||||
'cloud-archive:%s' % uca_pocket])
|
|
||||||
run_local(['sudo', 'apt-get', 'update'])
|
|
||||||
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
|
|
||||||
if uca_pocket:
|
|
||||||
run_local(['sudo', 'rm', '-f',
|
|
||||||
'/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
|
|
||||||
run_local(['sudo', 'apt-get', 'update'])
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
distribution = sys.argv[1]
|
|
||||||
|
|
||||||
branches = local_prep(distribution)
|
|
||||||
image_filenames = []
|
|
||||||
for branch_data in branches:
|
|
||||||
if branch_data.get('debs'):
|
|
||||||
cache_debs(branch_data['debs'])
|
|
||||||
for uca in sorted(UCA_POCKETS):
|
|
||||||
cache_debs(branch_data['debs'], uca)
|
|
||||||
elif branch_data.get('rpms'):
|
|
||||||
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
|
|
||||||
branch_data['rpms'])
|
|
||||||
else:
|
|
||||||
sys.exit('No supported package data found.')
|
|
||||||
|
|
||||||
for url in branch_data['images']:
|
|
||||||
fname = url.split('/')[-1]
|
|
||||||
if fname in image_filenames:
|
|
||||||
continue
|
|
||||||
image_filenames.append(fname)
|
|
||||||
download(url, fname)
|
|
||||||
|
|
||||||
# cache get-pip, because upstream network connection fails more
|
|
||||||
# often than you might imagine.
|
|
||||||
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
#
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import os.path
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import sys
|
|
||||||
import urllib2
|
|
||||||
|
|
||||||
from common import run_local
|
|
||||||
|
|
||||||
URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
|
|
||||||
'plain/gerrit/projects.yaml')
|
|
||||||
PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$')
|
|
||||||
|
|
||||||
# Not using an arg libraries in order to avoid module imports that
|
|
||||||
# are not available across all python versions
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
GIT_BASE = sys.argv[1]
|
|
||||||
else:
|
|
||||||
GIT_BASE = 'git://git.openstack.org'
|
|
||||||
|
|
||||||
|
|
||||||
def clone_repo(project):
|
|
||||||
remote = '%s/%s.git' % (GIT_BASE, project)
|
|
||||||
|
|
||||||
# Clear out any existing target directory first, in case of a retry.
|
|
||||||
try:
|
|
||||||
shutil.rmtree(os.path.join('/opt/git', project))
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Try to clone the requested git repository.
|
|
||||||
(status, out) = run_local(['git', 'clone', remote, project],
|
|
||||||
status=True, cwd='/opt/git')
|
|
||||||
|
|
||||||
# If it claims to have worked, make sure we can list branches.
|
|
||||||
if status == 0:
|
|
||||||
(status, moreout) = run_local(['git', 'branch', '-a'], status=True,
|
|
||||||
cwd=os.path.join('/opt/git', project))
|
|
||||||
out = '\n'.join((out, moreout))
|
|
||||||
|
|
||||||
# If that worked, try resetting to HEAD to make sure it's there.
|
|
||||||
if status == 0:
|
|
||||||
(status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
|
|
||||||
status=True,
|
|
||||||
cwd=os.path.join('/opt/git', project))
|
|
||||||
out = '\n'.join((out, moreout))
|
|
||||||
|
|
||||||
# Status of 0 imples all the above worked, 1 means something failed.
|
|
||||||
return (status, out)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
# TODO(jeblair): use gerrit rest api when available
|
|
||||||
data = urllib2.urlopen(URL).read()
|
|
||||||
for line in data.split('\n'):
|
|
||||||
# We're regex-parsing YAML so that we don't have to depend on the
|
|
||||||
# YAML module which is not in the stdlib.
|
|
||||||
m = PROJECT_RE.match(line)
|
|
||||||
if m:
|
|
||||||
project = m.group(1)
|
|
||||||
dirname = os.path.dirname(project)
|
|
||||||
# Skip repos that are inactive
|
|
||||||
if not ('attic' in dirname or dirname == 'stackforge'):
|
|
||||||
(status, out) = clone_repo(project)
|
|
||||||
print out
|
|
||||||
if status != 0:
|
|
||||||
print 'Retrying to clone %s' % m.group(1)
|
|
||||||
(status, out) = clone_repo(m.group(1))
|
|
||||||
print out
|
|
||||||
if status != 0:
|
|
||||||
raise Exception('Failed to clone %s' % m.group(1))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
#
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
|
|
||||||
def run_local(cmd, status=False, cwd='.', env={}):
|
|
||||||
print "Running:", cmd
|
|
||||||
newenv = os.environ
|
|
||||||
newenv.update(env)
|
|
||||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
|
|
||||||
stderr=subprocess.STDOUT, env=newenv)
|
|
||||||
(out, nothing) = p.communicate()
|
|
||||||
if status:
|
|
||||||
return (p.returncode, out.strip())
|
|
||||||
return out.strip()
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
#!/bin/bash -xe
|
|
||||||
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
#
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
# Set the grub timeout to 0.
|
|
||||||
if [ -f /etc/default/grub ] ; then
|
|
||||||
sudo sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' \
|
|
||||||
/etc/default/grub
|
|
||||||
if which update-grub &> /dev/null ; then
|
|
||||||
sudo update-grub
|
|
||||||
else
|
|
||||||
# If update-grub isn't available, use grub2-mkconfig directly
|
|
||||||
sudo /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
|
|
||||||
fi
|
|
||||||
elif [ -f /boot/grub/grub.conf ] ; then
|
|
||||||
sudo sed -i -e 's/^timeout=[0-9]\+/timeout=0/' \
|
|
||||||
/boot/grub/grub.conf
|
|
||||||
fi
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
#!/bin/bash -xe
|
|
||||||
|
|
||||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
#
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
mkdir -p ~/cache/files
|
|
||||||
mkdir -p ~/cache/pip
|
|
||||||
|
|
||||||
if [ -f /usr/bin/yum ]; then
|
|
||||||
sudo yum -y install python-devel python3-devel make automake gcc gcc-c++ \
|
|
||||||
kernel-devel redhat-lsb-core
|
|
||||||
elif [ -f /usr/bin/apt-get ]; then
|
|
||||||
if [ "$(lsb_release -c -s)" = "precise" ]; then
|
|
||||||
# temporary - remove the breaking -70 kernel and
|
|
||||||
# reinstall -69 to prevent openvswitch breakage
|
|
||||||
sudo DEBIAN_FRONTEND=noninteractive apt-get \
|
|
||||||
--option "Dpkg::Options::=--force-confold" \
|
|
||||||
--assume-yes remove linux-headers-3.2.0-70 \
|
|
||||||
linux-headers-3.2.0-70-virtual \
|
|
||||||
linux-image-3.2.0-70-virtual \
|
|
||||||
linux-headers-3.2.0-70-generic \
|
|
||||||
linux-image-3.2.0-70-generic
|
|
||||||
sudo DEBIAN_FRONTEND=noninteractive apt-get \
|
|
||||||
--option "Dpkg::Options::=--force-confold" \
|
|
||||||
--assume-yes install --reinstall linux-headers-3.2.0-69 \
|
|
||||||
linux-headers-3.2.0-69-virtual \
|
|
||||||
linux-image-3.2.0-69-virtual \
|
|
||||||
python-software-properties build-essential python-dev python3-dev
|
|
||||||
else
|
|
||||||
sudo DEBIAN_FRONTEND=noninteractive apt-get \
|
|
||||||
--option "Dpkg::Options::=--force-confold" \
|
|
||||||
--assume-yes install build-essential python-dev python3-dev \
|
|
||||||
python-software-properties linux-headers-virtual \
|
|
||||||
linux-headers-$(uname -r)
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Unsupported distro."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
Reference in New Issue
Block a user