Cleanup image build playbooks, elements, and roles
These are in zuul-providers now. Change-Id: Ic07b680932e033a3cd360038c273d030ffe6cfdc
This commit is contained in:
James E. Blair
parent
5251425216
commit
675974dc31
dib-elements
README.rstbindep-fallback.txt
cache-devstack
README.rstelement-depssource-repository-dstat_graphsource-repository-imagessource-repository-pipsource-repository-stackvizsource-repository-zanatacli
control-plane-minimal
infra-package-needs
initialize-urandom
nodepool-base
README.rst
cleanup.d
element-depsenvironment.d
finalise.d
glean@.service.d
install.d
05-record-details06-record-builddate20-iptables50-disable-metadata-cloudinit91-venv-os-testr96-clean-cron99-disable-rfc3041
package-installs.yamlpkg-mappost-install.d
pre-install.d
root.d
openstack-repos
zuul-worker
playbooks/opendev-build-diskimage-base
roles
image-upload-swift
make-source-repositories-cache
@ -1,74 +0,0 @@
|
||||
Using diskimage-builder to build opendev-ci nodes
|
||||
====================================================
|
||||
|
||||
In addition to being able to just download and consume images that are the
|
||||
same as what run devstack, it's easy to make your own for local dev or
|
||||
testing - or just for fun.
|
||||
|
||||
Install diskimage-builder
|
||||
-------------------------
|
||||
|
||||
Install the dependencies:
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get install kpartx qemu-utils curl python-yaml debootstrap
|
||||
|
||||
Install diskimage-builder:
|
||||
|
||||
::
|
||||
|
||||
sudo -H pip install diskimage-builder
|
||||
|
||||
|
||||
Build an image
|
||||
--------------
|
||||
|
||||
Building an image is simple, we have a script!
|
||||
|
||||
::
|
||||
|
||||
bash tools/build-image.sh
|
||||
|
||||
See the script for environment variables to set distribution, etc. By default
|
||||
it builds an ubuntu-minimal based image. You should be left with a .qcow2
|
||||
image file of your selected distribution.
|
||||
|
||||
Infra uses the -minimal build type for building Ubuntu/CentOS/Fedora. For
|
||||
example: ubuntu-minimal.
|
||||
|
||||
It is a good idea to set ``TMP_DIR`` to somewhere with plenty of space
|
||||
to avoid the disappointment of a full-disk mid-way through the script
|
||||
run.
|
||||
|
||||
While testing, consider exporting DIB_OFFLINE=true, to skip updating the cache.
|
||||
|
||||
Mounting the image
|
||||
------------------
|
||||
|
||||
If you would like to examine the contents of the image, you can mount it on
|
||||
a loopback device using qemu-nbd.
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get install qemu-utils
|
||||
sudo modprobe nbd max_part=16
|
||||
sudo mkdir -p /tmp/newimage
|
||||
sudo qemu-nbd -c /dev/nbd1 /path/to/opendev-ci-node-precise.qcow2
|
||||
sudo mount /dev/nbd1p1 /tmp/newimage
|
||||
|
||||
or use the scripts
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get install qemu-utils
|
||||
sudo modprobe nbd max_part=16
|
||||
sudo tools/mount-image.sh opendev-ci-node-precise.qcow2
|
||||
sudo tools/umount-image.sh
|
||||
|
||||
Other things
|
||||
------------
|
||||
|
||||
It's a qcow2 image, so you can do tons of things with it. You can upload it
|
||||
to glance, you can boot it using kvm, and you can even copy it to a cloud
|
||||
server, replace the contents of the server with it and kexec the new kernel.
|
||||
@ -1,114 +0,0 @@
|
||||
# This is the fallback list for packages to install. Do not add
|
||||
# additional packages here. Repositories should use bindep and create
|
||||
# their own bindep.txt files if the list below is not
|
||||
# working for them.
|
||||
|
||||
build-essential [platform:dpkg]
|
||||
curl [!platform:gentoo]
|
||||
net-misc/curl [platform:gentoo]
|
||||
dev-libs/cyrus-sasl [platform:gentoo]
|
||||
cyrus-sasl-devel [platform:rpm]
|
||||
media-fonts/nanumfont [platform:gentoo]
|
||||
fonts-nanum [platform:dpkg]
|
||||
media-fonts/takao-fonts [platform:gentoo]
|
||||
fonts-takao [platform:dpkg]
|
||||
gawk
|
||||
gettext [!platform:suse]
|
||||
gettext-runtime [platform:suse]
|
||||
graphviz [!platform:gentoo]
|
||||
media-gfx/graphviz [platform:gentoo]
|
||||
language-pack-en [platform:ubuntu]
|
||||
libcurl-devel [platform:rpm]
|
||||
libcurl4-gnutls-dev [platform:dpkg]
|
||||
libevent-dev [platform:dpkg]
|
||||
libevent-devel [platform:rpm]
|
||||
dev-libs/libevent [platform:gentoo]
|
||||
libffi-dev [platform:dpkg]
|
||||
libffi-devel [platform:redhat platform:suse]
|
||||
virtual/libffi [platform:gentoo]
|
||||
libjerasure-dev [platform:ubuntu]
|
||||
dev-libs/jerasure [platform:gentoo]
|
||||
libjpeg-dev [platform:dpkg]
|
||||
libjpeg8-devel [platform:suse]
|
||||
libjpeg-turbo-devel [platform:redhat]
|
||||
media-libs/libjpeg-turbo [platform:gentoo]
|
||||
libldap2-dev [platform:dpkg]
|
||||
net-nds/openldap [platform:gentoo]
|
||||
libmysqlclient-dev [platform:dpkg]
|
||||
libpcap-dev [platform:dpkg]
|
||||
libpcap-devel [platform:rpm]
|
||||
net-libs/libpcap [platform:gentoo]
|
||||
libpq-dev [platform:dpkg]
|
||||
librrd-dev [platform:dpkg]
|
||||
net-analyzer/rrdtool [platform:gentoo]
|
||||
libsasl2-dev [platform:dpkg]
|
||||
libselinux-python [platform:redhat]
|
||||
python-selinux [platform:suse]
|
||||
sys-libs/libselinux [platform:gentoo]
|
||||
libsqlite3-dev [platform:dpkg]
|
||||
libuuid-devel [platform:rpm]
|
||||
libvirt-dev [platform:dpkg]
|
||||
libvirt-devel [platform:rpm]
|
||||
app-emulation/libvirt [platform:gentoo]
|
||||
libvirt-python [platform:rpm !platform:suse]
|
||||
dev-python/libvirt-python [platform:gentoo]
|
||||
libxml2-dev [platform:dpkg]
|
||||
libxml2-devel [platform:rpm]
|
||||
libxml2-utils [platform:dpkg]
|
||||
dev-libs/libxml2 [platform:gentoo]
|
||||
libxslt-devel [platform:rpm]
|
||||
libxslt1-dev [platform:dpkg]
|
||||
dev-libs/libxslt [platform:gentoo]
|
||||
locales [platform:debian]
|
||||
mariadb [platform:rpm]
|
||||
mariadb-server [platform:redhat]
|
||||
mariadb-devel [platform:redhat]
|
||||
libmysqlclient-devel [platform:suse]
|
||||
dev-db/mariadb [platform:gentoo]
|
||||
memcached
|
||||
mysql-client [platform:dpkg]
|
||||
mysql-server [platform:dpkg]
|
||||
openldap-devel [platform:redhat]
|
||||
openldap2-devel [platform:suse]
|
||||
pkg-config [platform:dpkg platform:suse]
|
||||
pkgconfig [platform:redhat]
|
||||
virtual/pkgconfig [platform:gentoo]
|
||||
postgresql
|
||||
postgresql-client [platform:dpkg]
|
||||
postgresql-devel [platform:rpm]
|
||||
postgresql-server [platform:rpm]
|
||||
pypy [platform:ubuntu]
|
||||
pypy-dev [platform:ubuntu]
|
||||
python-dev [platform:dpkg]
|
||||
python-devel [platform:rpm]
|
||||
dev-lang/python [platform:gentoo]
|
||||
python-libvirt [platform:dpkg]
|
||||
python-lxml [!platform:gentoo !platform:fedora]
|
||||
python2-lxml [platform:fedora]
|
||||
dev-python/lxml [platform:gentoo]
|
||||
# Note that python3-all-dev includes python3-all, added
|
||||
# both here for documentary purpose.
|
||||
python3-all [platform:dpkg]
|
||||
python3-all-dev [platform:dpkg]
|
||||
python3-devel [platform:fedora platform:suse]
|
||||
# python3-devel does not pull in the python3 package on openSUSE so
|
||||
# we need to be explicit. The python3 package contains the XML module
|
||||
# which is required by a python3 virtualenv. Similarly, in python2,
|
||||
# the XML module is located in python-xml which is not pulled in
|
||||
# by python-devel as well. See https://bugzilla.suse.com/show_bug.cgi?id=1046990
|
||||
python3 [platform:suse]
|
||||
python-xml [platform:suse]
|
||||
rrdtool-devel [platform:rpm]
|
||||
sqlite [platform:redhat]
|
||||
sqlite-devel [platform:redhat]
|
||||
sqlite3-devel [platform:suse]
|
||||
sqlite3 [platform:dpkg]
|
||||
dev-db/sqlite [platform:gentoo]
|
||||
swig
|
||||
unzip
|
||||
uuid-dev [platform:dpkg]
|
||||
xsltproc [platform:dpkg]
|
||||
zip
|
||||
zlib-devel [platform:rpm]
|
||||
zlib1g-dev [platform:dpkg]
|
||||
sys-libs/zlib [platform:gentoo]
|
||||
@ -1,17 +0,0 @@
|
||||
cache-devstack
|
||||
==============
|
||||
|
||||
Pre-cache a range of things into CI images. This element uses the
|
||||
``source-repositories`` element to acquire files to be cached. The
|
||||
standard cache location is ``/opt/cache/files``.
|
||||
|
||||
A number of strategies are used to get the files to be cached.
|
||||
|
||||
We have a number of ``source-repository-*`` files for each package
|
||||
package that should be cached into images.
|
||||
|
||||
``extra-data.d/55-cache-devstack-repos`` goes through each devstack
|
||||
branch and runs the ``tools/image_list.sh`` script to dynamically
|
||||
build a list of files to cache as requested by devstack. This is
|
||||
mostly virtual machine images, but also some other peripheral packages.
|
||||
|
||||
@ -1 +0,0 @@
|
||||
openstack-repos
|
||||
@ -1 +0,0 @@
|
||||
dstat_graph git /opt/cache/dstat_graph/ https://opendev.org/opendev/dstat_graph master
|
||||
@ -1,15 +0,0 @@
|
||||
cirros-0.5.2-x86_64-disk.img file /opt/cache/files/cirros-0.5.2-x86_64-disk.img https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img
|
||||
cirros-0.5.2-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.5.2-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-uec.tar.gz
|
||||
cirros-0.5.2-aarch64-disk.img file /opt/cache/files/cirros-0.5.2-aarch64-disk.img https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-aarch64-disk.img
|
||||
cirros-0.5.3-x86_64-disk.img file /opt/cache/files/cirros-0.5.3-x86_64-disk.img https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-disk.img
|
||||
cirros-0.5.3-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.5.3-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-uec.tar.gz
|
||||
cirros-0.5.3-aarch64-disk.img file /opt/cache/files/cirros-0.5.3-aarch64-disk.img https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-aarch64-disk.img
|
||||
cirros-0.6.1-x86_64-disk.img file /opt/cache/files/cirros-0.6.1-x86_64-disk.img https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img
|
||||
cirros-0.6.1-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.6.1-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-uec.tar.gz
|
||||
cirros-0.6.1-aarch64-disk.img file /opt/cache/files/cirros-0.6.1-aarch64-disk.img https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-aarch64-disk.img
|
||||
cirros-0.6.2-x86_64-disk.img file /opt/cache/files/cirros-0.6.2-x86_64-disk.img https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img
|
||||
cirros-0.6.2-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.6.2-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-uec.tar.gz
|
||||
cirros-0.6.2-aarch64-disk.img file /opt/cache/files/cirros-0.6.2-aarch64-disk.img https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-aarch64-disk.img
|
||||
etcd-v3.1.10-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.1.10-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.1.10/etcd-v3.1.10-linux-amd64.tar.gz
|
||||
etcd-v3.2.17-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.2.17-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.2.17/etcd-v3.2.17-linux-amd64.tar.gz
|
||||
etcd-v3.3.12-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.3.12-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.3.12/etcd-v3.3.12-linux-amd64.tar.gz
|
||||
@ -1,2 +0,0 @@
|
||||
get-pip-py file /opt/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py
|
||||
get-pip-py-py36 file /opt/cache/files/get-pip.py-py36 https://bootstrap.pypa.io/pip/3.6/get-pip.py
|
||||
@ -1 +0,0 @@
|
||||
stackviz file /opt/cache/files/stackviz-latest.tar.gz https://tarballs.openstack.org/stackviz/dist/stackviz-latest.tar.gz
|
||||
@ -1 +0,0 @@
|
||||
zanata-cli file /opt/cache/files/zanata-cli-4.3.3-dist.tar.gz https://search.maven.org/remotecontent?filepath=org/zanata/zanata-cli/4.3.3/zanata-cli-4.3.3-dist.tar.gz
|
||||
@ -1,4 +0,0 @@
|
||||
control-plane-minimal
|
||||
---------------------
|
||||
|
||||
Essential packages for an OpenDev infra control-plane image.
|
||||
@ -1,2 +0,0 @@
|
||||
openssh-server
|
||||
package-installs
|
||||
@ -1,11 +0,0 @@
|
||||
acpid:
|
||||
coreutils:
|
||||
cron:
|
||||
util-linux:
|
||||
python3:
|
||||
phase: pre-install.d
|
||||
ntp:
|
||||
ntpdate:
|
||||
lvm2:
|
||||
haveged:
|
||||
iptables:
|
||||
@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
systemd)
|
||||
systemctl enable haveged.service
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported init system $DIB_INIT_SYSTEM"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
systemd)
|
||||
systemctl enable ntp.service
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported init system $DIB_INIT_SYSTEM"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
# Copyright 2016 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# NOTE(pabelanger): Glean configures access for root user, so allow us to
|
||||
# properly login.
|
||||
sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \
|
||||
&& echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
|
||||
|
||||
# NOTE(clarkb): Glean configures ssh keys only and not passwords. Disable
|
||||
# unnecessary password auth.
|
||||
sed -i -e '/PasswordAuthentication/d' /etc/ssh/sshd_config \
|
||||
&& echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
|
||||
@ -1,4 +0,0 @@
|
||||
infra-package-needs
|
||||
-------------------
|
||||
|
||||
Install required packages for a OpenDev infra host
|
||||
@ -1,3 +0,0 @@
|
||||
ensure-venv
|
||||
openssh-server
|
||||
package-installs
|
||||
@ -1,27 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
if [[ "$DISTRO_NAME" =~ ^(centos|rocky)$ && ! ${DIB_RELEASE} =~ '9' ]] ; then
|
||||
# 9-stream we are just using the default rngd
|
||||
# Note: $YUM exposed by centos|rhel-like environment, correct across releases
|
||||
${YUM} -y install --enablerepo=epel haveged
|
||||
fi
|
||||
@ -1,33 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2015 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# Install bindep into a virtualenv
|
||||
# This is in /usr instead of /usr/local due to this bug on precise:
|
||||
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
python3 -m venv /usr/bindep-env
|
||||
if [[ ${DIB_RELEASE} == 'xenial' ]]; then
|
||||
# The pip on xenial can't figure out it shouldn't install
|
||||
# the latest pip; this is the last to support 3.5
|
||||
/usr/bindep-env/bin/pip install --upgrade pip==20.3.4
|
||||
else
|
||||
/usr/bindep-env/bin/pip install --upgrade pip
|
||||
fi
|
||||
/usr/bindep-env/bin/pip install bindep
|
||||
@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
if [ "$DISTRO_NAME" == "ubuntu" ]; then
|
||||
rsyslog_dir="$(dirname $0)/../rsyslog.d"
|
||||
cp -RP $rsyslog_dir/* /etc/rsyslog.d/
|
||||
fi
|
||||
@ -1,28 +0,0 @@
|
||||
acl:
|
||||
acpid:
|
||||
coreutils:
|
||||
cron:
|
||||
util-linux:
|
||||
build-essential:
|
||||
python3-dev:
|
||||
uuid-runtime:
|
||||
traceroute:
|
||||
ntp:
|
||||
ntpdate:
|
||||
gentoolkit:
|
||||
at:
|
||||
strace:
|
||||
tcpdump:
|
||||
rsyslog:
|
||||
git:
|
||||
rsync:
|
||||
parted:
|
||||
wget:
|
||||
iputils-ping:
|
||||
iproute2:
|
||||
dnsutils:
|
||||
haveged:
|
||||
iptables:
|
||||
redhat-rpm-config:
|
||||
redhat-lsb-core:
|
||||
gnupg2:
|
||||
@ -1,117 +0,0 @@
|
||||
{
|
||||
"release": {
|
||||
"centos": {
|
||||
"8": {
|
||||
"ntp": "chrony",
|
||||
"ntpdate": ""
|
||||
},
|
||||
"9-stream": {
|
||||
"ntp": "chrony",
|
||||
"ntpdate": "",
|
||||
"iptables": "iptables-services"
|
||||
}
|
||||
},
|
||||
"debian": {
|
||||
"bookworm": {
|
||||
"ntp": "",
|
||||
"ntpdate": ""
|
||||
}
|
||||
},
|
||||
"rocky": {
|
||||
"8": {
|
||||
"ntp": "chrony",
|
||||
"ntpdate": "",
|
||||
"coreutils": ""
|
||||
},
|
||||
"9": {
|
||||
"ntp": "chrony",
|
||||
"ntpdate": "",
|
||||
"iptables": "iptables-services",
|
||||
"coreutils": ""
|
||||
}
|
||||
},
|
||||
"ubuntu": {
|
||||
"focal": {
|
||||
"ntp": "",
|
||||
"ntpdate": ""
|
||||
},
|
||||
"jammy": {
|
||||
"ntp": "",
|
||||
"ntpdate": ""
|
||||
},
|
||||
"noble": {
|
||||
"ntp": "",
|
||||
"ntpdate": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
"distro": {
|
||||
"fedora": {
|
||||
"python3-dev": "python3-devel",
|
||||
"iptables": "iptables-services",
|
||||
"haveged": "haveged",
|
||||
"ntp": "chrony",
|
||||
"ntpdate": "",
|
||||
"redhat-rpm-config": "redhat-rpm-config",
|
||||
"redhat-lsb-core": "redhat-lsb-core"
|
||||
},
|
||||
"openeuler": {
|
||||
"cron": "cronie",
|
||||
"build-essential": "glibc-devel gcc make",
|
||||
"dnsutils": "bind-utils",
|
||||
"iproute2": "iproute",
|
||||
"iputils-ping": "iputils",
|
||||
"ntp": "ntp ntp-perl",
|
||||
"python3-dev": "python3-devel",
|
||||
"iptables": "iptables-services",
|
||||
"haveged": "haveged"
|
||||
},
|
||||
"gentoo": {
|
||||
"acl": "sys-apps/acl",
|
||||
"build-essential": "",
|
||||
"cron": "sys-process/cronie",
|
||||
"dnsutils": "net-dns/bind-tools",
|
||||
"gentoolkit": "app-portage/gentoolkit",
|
||||
"git": "dev-vcs/git",
|
||||
"gnupg2": "app-crypt/gnupg",
|
||||
"iptables": "net-firewall/iptables",
|
||||
"iputils-ping": "net-misc/iputils",
|
||||
"ntp": "net-misc/ntp",
|
||||
"ntpdate": "net-misc/ntp",
|
||||
"python3-dev": "",
|
||||
"traceroute": "net-analyzer/traceroute",
|
||||
"uuid-runtime": ""
|
||||
}
|
||||
},
|
||||
"family": {
|
||||
"redhat": {
|
||||
"cron": "cronie",
|
||||
"build-essential": "glibc-devel gcc make",
|
||||
"dnsutils": "bind-utils",
|
||||
"iproute2": "iproute",
|
||||
"iputils-ping": "iputils",
|
||||
"ntp": "ntp ntp-perl",
|
||||
"python3-dev": "",
|
||||
"uuid-runtime": "",
|
||||
"iptables": "iptables-services",
|
||||
"haveged": ""
|
||||
},
|
||||
"suse": {
|
||||
"build-essential": "glibc-devel gcc make",
|
||||
"dnsutils": "bind-utils",
|
||||
"git": "git-core",
|
||||
"gnupg2": "gpg2",
|
||||
"iputils-ping": "iputils",
|
||||
"ntpdate": "",
|
||||
"python3-dev": "python3-devel",
|
||||
"iptables": "iptables",
|
||||
"uuid-runtime": "uuidd"
|
||||
}
|
||||
},
|
||||
"default": {
|
||||
"iptables": "iptables-persistent",
|
||||
"gentoolkit": "",
|
||||
"redhat-rpm-config": "",
|
||||
"redhat-lsb-core": ""
|
||||
}
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
if [[ "$DISTRO_NAME" =~ ^(centos|rocky)$ && ${DIB_RELEASE} =~ '9' ]]; then
|
||||
# 9/9-stream we are just using the default rngd
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
upstart)
|
||||
# nothing to do
|
||||
exit 0
|
||||
;;
|
||||
systemd)
|
||||
systemctl enable haveged.service
|
||||
;;
|
||||
openrc)
|
||||
rc-update add haveged default
|
||||
;;
|
||||
sysv)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported init system $DIB_INIT_SYSTEM"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@ -1,38 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
upstart)
|
||||
# nothing to do
|
||||
exit 0
|
||||
;;
|
||||
systemd)
|
||||
# stick with default systemd timesyncd on bookworm, focal and beyond
|
||||
if [[ ":bookworm: :focal: :jammy: :noble:" =~ :${DIB_RELEASE}: ]]; then
|
||||
exit 0
|
||||
elif [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
|
||||
systemctl enable ntp.service
|
||||
elif [[ ( $DISTRO_NAME == "centos" && $DIB_RELEASE > 7 ) || $DISTRO_NAME == "fedora" || $DISTRO_NAME == "rocky" ]]; then
|
||||
systemctl enable chronyd
|
||||
else
|
||||
systemctl enable ntpd.service
|
||||
fi
|
||||
;;
|
||||
openrc)
|
||||
rc-update add ntp-client default
|
||||
rc-update add acpid default
|
||||
;;
|
||||
sysv)
|
||||
# ntp is enabled by default, nothing to do
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported init system $DIB_INIT_SYSTEM"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
# Copyright 2016 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# NOTE(pabelanger): Glean configures access for root user, so allow us to
|
||||
# properly login.
|
||||
sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \
|
||||
&& echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
|
||||
|
||||
# NOTE(clarkb): Glean configures ssh keys only and not passwords. Disable
|
||||
# unnecessary password auth.
|
||||
sed -i -e '/PasswordAuthentication/d' /etc/ssh/sshd_config \
|
||||
&& echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
|
||||
|
||||
# NOTE(clarkb): SSH scanners may be affecting Zuul ssh connectivity
|
||||
# Default LoginGraceTime is 120. Reduce that to 30 to cycle connections more
|
||||
# quickly.
|
||||
sed -i -e '/LoginGraceTime/d' /etc/ssh/sshd_config \
|
||||
&& echo "LoginGraceTime 30" >> /etc/ssh/sshd_config
|
||||
|
||||
# NOTE(clarkb): SSH scanners may be affecting Zuul ssh connectivity
|
||||
# Default MaxStartups is 10:30:100 which means after 10 unauthenticated
|
||||
# connections randomly drop 30% of connections with an increasing
|
||||
# percentage until 100 connections is reached.
|
||||
sed -i -e '/MaxStartups/d' /etc/ssh/sshd_config \
|
||||
&& echo "MaxStartups 30:10:100" >> /etc/ssh/sshd_config
|
||||
@ -1,18 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# needed for unbound
|
||||
if [ "${DISTRO_NAME}" == "gentoo" ]; then
|
||||
mkdir -p /etc/portage/package.use
|
||||
echo "app-admin/rsyslog openssl" >> /etc/portage/package.use/rsyslog
|
||||
echo "dev-lang/python sqlite" >> /etc/portage/package.use/python
|
||||
echo "dev-libs/openssl static-libs -bindist" >> /etc/portage/package.use/openssl
|
||||
echo "net-misc/openssh -bindist" >> /etc/portage/package.use/openssh
|
||||
echo "sys-apps/systemd audit curl gcrypt importd lzma nat" >> /etc/portage/package.use/systemd
|
||||
echo "sys-libs/zlib static-libs" >> /etc/portage/package.use/zlib
|
||||
fi
|
||||
@ -1,68 +0,0 @@
|
||||
# Default rules for rsyslog.
|
||||
#
|
||||
# For more information see rsyslog.conf(5) and /etc/rsyslog.conf
|
||||
|
||||
#
|
||||
# First some standard log files. Log by facility.
|
||||
#
|
||||
auth,authpriv.* /var/log/auth.log
|
||||
*.*;auth,authpriv.none -/var/log/syslog
|
||||
#cron.* /var/log/cron.log
|
||||
#daemon.* -/var/log/daemon.log
|
||||
kern.* -/var/log/kern.log
|
||||
#lpr.* -/var/log/lpr.log
|
||||
mail.* -/var/log/mail.log
|
||||
#user.* -/var/log/user.log
|
||||
|
||||
#
|
||||
# Logging for the mail system. Split it up so that
|
||||
# it is easy to write scripts to parse these files.
|
||||
#
|
||||
#mail.info -/var/log/mail.info
|
||||
#mail.warn -/var/log/mail.warn
|
||||
mail.err /var/log/mail.err
|
||||
|
||||
#
|
||||
# Logging for INN news system.
|
||||
#
|
||||
news.crit /var/log/news/news.crit
|
||||
news.err /var/log/news/news.err
|
||||
news.notice -/var/log/news/news.notice
|
||||
|
||||
#
|
||||
# Some "catch-all" log files.
|
||||
#
|
||||
#*.=debug;\
|
||||
# auth,authpriv.none;\
|
||||
# news.none;mail.none -/var/log/debug
|
||||
#*.=info;*.=notice;*.=warn;\
|
||||
# auth,authpriv.none;\
|
||||
# cron,daemon.none;\
|
||||
# mail,news.none -/var/log/messages
|
||||
|
||||
#
|
||||
# Emergencies are sent to everybody logged in.
|
||||
#
|
||||
*.emerg :omusrmsg:*
|
||||
|
||||
#
|
||||
# I like to have messages displayed on the console, but only on a virtual
|
||||
# console I usually leave idle.
|
||||
#
|
||||
#daemon,mail.*;\
|
||||
# news.=crit;news.=err;news.=notice;\
|
||||
# *.=debug;*.=info;\
|
||||
# *.=notice;*.=warn /dev/tty8
|
||||
|
||||
# The named pipe /dev/xconsole is for the `xconsole' utility. To use it,
|
||||
# you must invoke `xconsole' with the `-file' option:
|
||||
#
|
||||
# $ xconsole -file /dev/xconsole [...]
|
||||
#
|
||||
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
|
||||
# busy site..
|
||||
#
|
||||
#daemon.*;mail.*;\
|
||||
# news.err;\
|
||||
# *.=debug;*.=info;\
|
||||
# *.=notice;*.=warn |/dev/xconsole
|
||||
@ -1,2 +0,0 @@
|
||||
This uses haveged to quickly initialize the nonblocking kernel random
|
||||
number generator at boot.
|
||||
@ -1,2 +0,0 @@
|
||||
dib-init-system
|
||||
install-static
|
||||
@ -1,13 +0,0 @@
|
||||
[Unit]
|
||||
Description=Quickly initialize the nonblocking kernel random number generator at boot.
|
||||
Before=network-pre.target
|
||||
Wants=network-pre.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
ExecStart=/usr/local/bin/initialize-urandom.py
|
||||
RemainAfterExit=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,18 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
systemd)
|
||||
sudo chmod 0644 /usr/lib/systemd/system/initialize-urandom.service
|
||||
systemctl enable initialize-urandom.service
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported init system"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@ -1,273 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2016 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import ctypes
|
||||
import errno
|
||||
import fcntl
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
|
||||
"""Add entropy to the kernel until the nonblocking pool is
|
||||
initialized.
|
||||
|
||||
The Linux kernel has 3 entropy pools: input, blocking, and
|
||||
nonblocking. Normally entropy accumulates in the input pool and as it
|
||||
is depleted by the other pools, it is transferred from the input pool
|
||||
to the others.
|
||||
|
||||
The blocking pool corresponds to /dev/random, where reads from that
|
||||
device return random numbers only as long as there is sufficient
|
||||
entropy in the blocking pool. When that entropy is depleted, further
|
||||
reads from /dev/random block until it is replenished.
|
||||
|
||||
The nonblocking pool corresponds to /dev/urandom, where reads never
|
||||
block. Even if there is no entropy in the nonblocking pool, random
|
||||
numbers are still returned.
|
||||
|
||||
The algorithms in use in Linux 3.17 require 128 bits of entropy in
|
||||
order to initialize the random number generators associated with each
|
||||
pool. Naturally, reads from /dev/random will not return until the
|
||||
associated generator is initialized. Reads from /dev/urandom will not
|
||||
block -- even if the generator is not initialized. The kernel will
|
||||
output a notice[1] if this happens.
|
||||
|
||||
In order to avoid the situation where urandom is used when
|
||||
uninitialized, the kernel diverts entropy from timers and interrupts
|
||||
to the nonblocking pool (instead of the input pool) until it is
|
||||
initialized. In this way, as the system boots, the nonblocking pool
|
||||
accumulates entropy first, reducing the time period during which
|
||||
urandom might produce numbers from an uninitialized generator, and
|
||||
then the input and blocking pools are filled.
|
||||
|
||||
Beginning with Linux 3.17, the getrandom(2) syscall was added[2] so
|
||||
that user-space programs that generally would like to use /dev/urandom
|
||||
can do so without opening a file descriptor and, more relevant here,
|
||||
can ensure that they do so only after the generator is initialized
|
||||
(which otherwise is not possible with the /dev/urandom interface).
|
||||
|
||||
Unfortunately, programs which use this interface during early boot may
|
||||
need to wait some time for the nonblocking pool to accumulate enough
|
||||
entropy to initialize, and therefore for getrandom to return.
|
||||
Particularly in the case of a VM, this may take considerable time.
|
||||
|
||||
There are many methods of addressing this shortcoming:
|
||||
|
||||
* Store data from /dev/random at shutdown and use it to seed the
|
||||
entropy pool at the next boot. Most GNU/Linux distributions do
|
||||
this. On Ubuntu Xenial, this task is performed by systemd[3].
|
||||
Unfortunately, while writes to /dev/random (which is the method
|
||||
systemd uses to seed the system at boot) do add data to the pool,
|
||||
they do not increase the internal tracking of the amount of entropy
|
||||
in the pool. Therefore, for the purposes of determining whether the
|
||||
nonblocking pool has accumulated 128 bits of entropy, they are not
|
||||
counted.
|
||||
|
||||
* Use haveged to maintain a sufficient amount of entropy. Haveged can
|
||||
produce entropy very quickly, and when run at boot, will typically
|
||||
immediately fill the entropy pool. Haveged performs an ioctl
|
||||
operation on /dev/random rather than writing data to it, and this
|
||||
ioctl allows it to specify how much entropy the data it supplies
|
||||
contains. Therefore, unlike writes to /dev/random, ioctls do
|
||||
increment the entropy counter. Unfortunately, data from ioctls are
|
||||
*always* directed to the input pool. While entropy from timers and
|
||||
interrupts are diverted to the nonblocking pool to speed its
|
||||
initialization, data arriving from the ioctl instead end up in the
|
||||
input pool for later use.
|
||||
|
||||
When more entropy than is needed is supplied to the input pool, the
|
||||
kernel will preemptively transfer some of that entropy to the
|
||||
secondary (including nonblocking) pools. Since haveged supplies so
|
||||
much data on startup, some of this entropy should be able to spill
|
||||
over into the nonblocking pool to aid it in achieving the
|
||||
initialization threshold. Unfortunately, at the stage of early boot
|
||||
we are considering, the input pool's generator also has not been
|
||||
initialized. When the kernel receives a large amount of data from
|
||||
haveged over the ioctl, it pushes the input pool's generator over
|
||||
the 128 bit threshold, and initializes the input pool's generator.
|
||||
When a pool's generator is initialized, the entropy counter for that
|
||||
pool is reset to zero. This leaves no entropy to spill over to the
|
||||
nonblocking pool. Haveged is only able to see the entropy count for
|
||||
the input pool, and therefore is unaware that further contributions
|
||||
of entropy would aid (via spill-over) in seeding the nonblocking
|
||||
pool.
|
||||
|
||||
At this point it's worth discussing why the nonblocking pool is
|
||||
still not initialized despite a full input pool. When a secondary
|
||||
pool needs more entropy, it can pull from the input pool. However,
|
||||
there is a timer that only allows the nonblocking pool to withdraw
|
||||
entropy from the input pool every 60 seconds by default (this can be
|
||||
adjusted via proc). If something during very early boot reads data
|
||||
from /dev/urandom, a transfer (from the very likely empty) input
|
||||
pool is initiated, starting the timer that will prevent another
|
||||
transfer for 60 seconds, even if the input pool is later filled
|
||||
(such as by haveged). This means that even with haveged running at
|
||||
boot the delay due to a blocking getrandom(2) call may still be as
|
||||
long as 60 seconds.
|
||||
|
||||
* Use rng-tools for the same purpose as haveged. rng-tools operates
|
||||
in a similar manner to haveged, supplying entropic data to the
|
||||
kernel via ioctl. However, it does so in smaller chunks. This
|
||||
means that once the input pool's generator surpasses the 128 bit
|
||||
threshold for initialization, entropy from the next ioctl from
|
||||
rng-tools will be available to spill over to the nonblocking pool,
|
||||
and may be sufficient to initialize it.
|
||||
|
||||
Because of this behavior, use of rng-tools may cause getrandom(2) to
|
||||
return more quickly at boot, however, this may only happen due to a
|
||||
quirk of implementation and relies on some specific values and
|
||||
conditions for the amount of entropy in the input pool at the time
|
||||
it is run.
|
||||
|
||||
This program speeds initialization of the nonblocking pool by adding
|
||||
entropy to the input pool in small chunks. To determine when the
|
||||
nonblocking pool is initialized, it performs the nonblocking
|
||||
getrandom(2) syscall requesting one byte of random data. As long as
|
||||
the nonblocking pool is uninitialized, that call will fail and set
|
||||
errno to EAGAIN. In that case, the program reads 64 bytes of data
|
||||
from haveged and sends it to the kernel using the ioctl interface,
|
||||
then repeats this in a loop. That will cause entropy to accumulate in
|
||||
the input pool until it is initialized and reaches the spill-over
|
||||
threshold. Further data will accumulate in the nonblocking pool until
|
||||
it is initialized. Once that occurs, the getrandom(2) call will
|
||||
return successfully, and the program will exit the loop.
|
||||
|
||||
There are other ways this problem could be addressed (changes to
|
||||
haveged or rng-tools to support behavior like this, or changes to the
|
||||
kernel to direct entropy received via ioctl to the nonblocking pool
|
||||
during initialization), however, this problem is likely to be
|
||||
short-lived as the nonblocking generator is being replaced[4] in
|
||||
current kernel versions and should not suffer from the same problem.
|
||||
|
||||
[1] http://lxr.free-electrons.com/source/drivers/char/random.c?v=3.17#L1385
|
||||
[2] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\
|
||||
?id=c6e9d6f38894798696f23c8084ca7edbf16ee895
|
||||
[3] https://www.freedesktop.org/software/systemd/man/systemd-random-seed.\
|
||||
service.html
|
||||
[4] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\
|
||||
?id=e192be9d9a30555aae2ca1dc3aad37cba484cd4a
|
||||
"""
|
||||
|
||||
|
||||
class GeneratorNotInitializedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InterruptedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Pump(object):
|
||||
# How much data, in bytes, to move at once. 64 is the size of the
|
||||
# internal kernel buffer, so we match it.
|
||||
CHUNK_SIZE = 64
|
||||
|
||||
# The syscall number for getrandom(2).
|
||||
SYS_getrandom = 318
|
||||
|
||||
# The IOCTL to add entropy.
|
||||
OP_RNDADDENTROPY = 0x40085203
|
||||
|
||||
# Flags for getrandom:
|
||||
GRND_NONBLOCK = 0x0001 # Do not block
|
||||
GRND_RANDOM = 0x0002 # Use /dev/random instead of urandom
|
||||
|
||||
def __init__(self):
|
||||
# Use ctypes to invoke getrandom since it is not available in
|
||||
# python. os.urandom may call getrandom in some versions of
|
||||
# python3, however, the blocking on initialization behavior is
|
||||
# seen as a bug and so os.urandom will never block, even if
|
||||
# getrandom would. See http://bugs.python.org/issue26839
|
||||
self._getrandom = ctypes.CDLL(None, use_errno=True).syscall
|
||||
self._getrandom.restype = ctypes.c_long
|
||||
# The arguments are syscall number, void *buf,
|
||||
# size_t buflen, unsigned int flags.
|
||||
self._getrandom.argtypes = (ctypes.c_long, ctypes.c_void_p,
|
||||
ctypes.c_size_t, ctypes.c_uint)
|
||||
|
||||
def getrandom(self, length, random=False, nonblock=False):
|
||||
flags = 0
|
||||
if random:
|
||||
flags |= self.GRND_RANDOM
|
||||
if nonblock:
|
||||
flags |= self.GRND_NONBLOCK
|
||||
buf = ctypes.ARRAY(ctypes.c_char, length)()
|
||||
r = self._getrandom(self.SYS_getrandom, buf, len(buf), flags)
|
||||
if r == -1:
|
||||
err = ctypes.get_errno()
|
||||
if err == errno.EINVAL:
|
||||
raise Exception("getrandom: Invalid argument")
|
||||
elif err == errno.EFAULT:
|
||||
raise Exception("getrandom: Buffer is outside "
|
||||
"accessible address space")
|
||||
elif err == errno.EAGAIN:
|
||||
raise GeneratorNotInitializedError()
|
||||
elif err == errno.EINTR:
|
||||
raise InterruptedError()
|
||||
return buf[:r]
|
||||
|
||||
def isInitialized(self):
|
||||
# Read one byte from getrandom to determine whether the
|
||||
# nonblocking pool is initialized.
|
||||
try:
|
||||
r = self.getrandom(1, nonblock=True)
|
||||
if len(r) != 1:
|
||||
raise Exception("No data returned from getrandom")
|
||||
print("Nonblocking pool initialized")
|
||||
return True
|
||||
except GeneratorNotInitializedError:
|
||||
return False
|
||||
|
||||
def run(self):
|
||||
"""Move data from haveged to the kernel until the nonblocking pool is
|
||||
initialized.
|
||||
|
||||
"""
|
||||
if self.isInitialized():
|
||||
return
|
||||
|
||||
random_fd = os.open('/dev/random', os.O_RDWR)
|
||||
# Start haveged and tell it to supply unlimited data on
|
||||
# stdout, and print summary information.
|
||||
p = subprocess.Popen(['/usr/sbin/haveged', '-f', '-', '-n', '0',
|
||||
'-v', '1'],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
while not self.isInitialized():
|
||||
# Read a chunk from haveged.
|
||||
data = b''
|
||||
while len(data) < self.CHUNK_SIZE:
|
||||
data += p.stdout.read(self.CHUNK_SIZE - len(data))
|
||||
# The data structure is:
|
||||
# struct rand_pool_info {
|
||||
# int entropy_count;
|
||||
# int buf_size;
|
||||
# __u32 buf[0];
|
||||
# };
|
||||
arg = struct.pack('iis', len(data) * 8, len(data), data)
|
||||
print("Moving %s bytes" % len(data))
|
||||
fcntl.ioctl(random_fd, self.OP_RNDADDENTROPY, arg)
|
||||
# Now that the generator is initialized, stop haveged and
|
||||
# print the summary information.
|
||||
p.send_signal(2)
|
||||
p.stdout.read()
|
||||
print(p.stderr.read().decode('utf-8'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = Pump()
|
||||
p.run()
|
||||
@ -1,24 +0,0 @@
|
||||
=============
|
||||
nodepool-base
|
||||
=============
|
||||
|
||||
Tasks to deal with image metadata and other Nodepool cloud specific tweaks.
|
||||
|
||||
Name resolution
|
||||
---------------
|
||||
|
||||
The image should have the unbound DNS resolver package installed, the
|
||||
``nodepool-base`` element then configures it to forward DNS queries
|
||||
to:
|
||||
|
||||
``NODEPOOL_STATIC_NAMESERVER_V4``, default: ``1.0.0.1``
|
||||
``NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK``, default: ``8.8.8.8``.
|
||||
|
||||
If ``NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6`` is set to ``1`` then
|
||||
the following two servers will be configured as forwarders too
|
||||
|
||||
``NODEPOOL_STATIC_NAMESERVER_V6``, default: ``2606:4700:4700::1111``
|
||||
``NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK``, default: ``2001:4860:4860::8888``
|
||||
|
||||
Note externally setting either of these values implies
|
||||
``NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6=1``
|
||||
@ -1,23 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2017 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
sudo rm -rf $TARGET_ROOT/root/.cache
|
||||
@ -1,3 +0,0 @@
|
||||
ensure-venv
|
||||
package-installs
|
||||
zuul-worker
|
||||
@ -1,4 +0,0 @@
|
||||
export NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2606:4700:4700::1111}
|
||||
export NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-1.0.0.1}
|
||||
export NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888}
|
||||
export NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8}
|
||||
@ -1,254 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=set setu setpipefail indent
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
#
|
||||
# Note that in OpenStack infra, the configure-unbound role [1] that is
|
||||
# part of the base jobs will reconfigure unbound based on the host's
|
||||
# ipv6 support very early in the job setup. Thus the following
|
||||
# forwarder setup is only relevant to the initial boot and some parts
|
||||
# of the integration-tests before configure-unbound role is used.
|
||||
#
|
||||
# [1] https://opendev.org/opendev/base-jobs/src/branch/master/roles/configure-unbound
|
||||
#
|
||||
|
||||
NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-1.0.0.1}
|
||||
NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8}
|
||||
|
||||
# Explicitly setting a v6 nameserver implies you want ipv6
|
||||
if [[ -n ${NODEPOOL_STATIC_NAMESERVER_V6:-} || -n ${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK} ]]; then
|
||||
NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6=1
|
||||
fi
|
||||
|
||||
if [[ ${NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6:-0} == 1 ]]; then
|
||||
NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2606:4700:4700::1111}
|
||||
NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888}
|
||||
|
||||
dd of=/tmp/forwarding.conf <<EOF
|
||||
forward-zone:
|
||||
name: "."
|
||||
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V6
|
||||
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK
|
||||
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V4
|
||||
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK
|
||||
EOF
|
||||
else
|
||||
# Otherwise, only populate ipv4 nameservers here. We used to just
|
||||
# do both, but we found it unreliable on hosts/platforms without
|
||||
# ipv6 support (see
|
||||
# https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=4188).
|
||||
dd of=/tmp/forwarding.conf <<EOF
|
||||
forward-zone:
|
||||
name: "."
|
||||
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V4
|
||||
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK
|
||||
EOF
|
||||
fi
|
||||
|
||||
mv /tmp/forwarding.conf /etc/unbound/
|
||||
chown root:root /etc/unbound/forwarding.conf
|
||||
chmod a+r /etc/unbound/forwarding.conf
|
||||
|
||||
UNBOUND_LOGGING_CONF="\
|
||||
server:
|
||||
chroot: \"\"
|
||||
logfile: \"/var/log/unbound.log\"
|
||||
# Log debug messages
|
||||
verbosity: 5"
|
||||
echo "$UNBOUND_LOGGING_CONF" > /tmp/unbound-logging.conf
|
||||
|
||||
if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|opensuse|openeuler|rocky) ]] ; then
|
||||
UNBOUND_CONFD=/etc/unbound/conf.d
|
||||
elif [[ "$DISTRO_NAME" =~ 'gentoo' ]] ; then
|
||||
UNBOUND_CONFD=/etc/unbound/conf.d
|
||||
mkdir -p $UNBOUND_CONFD
|
||||
echo "include: \"$UNBOUND_CONFD/*.conf\"" >> /etc/unbound/unbound.conf
|
||||
else
|
||||
UNBOUND_CONFD=/etc/unbound/unbound.conf.d
|
||||
fi
|
||||
|
||||
mv /tmp/unbound-logging.conf $UNBOUND_CONFD
|
||||
chown root:root $UNBOUND_CONFD/unbound-logging.conf
|
||||
chmod a+r $UNBOUND_CONFD/unbound-logging.conf
|
||||
|
||||
touch /var/log/unbound.log
|
||||
chown unbound /var/log/unbound.log
|
||||
chmod 0644 /var/log/unbound.log
|
||||
# NOTE(ianw) 2022-05-22 : not 100% sure why but emperically unbound
|
||||
# runs under named context. Generally it doesn't log here on rhel-ish
|
||||
# distros, but for consistency it's much easier if we just have one
|
||||
# log file in the same place across distros. Thus set the context to
|
||||
# allow it.
|
||||
if [[ -e /usr/sbin/semanage ]]; then
|
||||
semanage fcontext -a -t named_log_t /var/log/unbound.log
|
||||
restorecon -v /var/log/unbound.log
|
||||
fi
|
||||
|
||||
if [[ "$DISTRO_NAME" =~ (opensuse) ]] ; then
|
||||
# NOTE(ianw) 2021-03-17 : something about building the suse chroot
|
||||
# has changed and /etc/init.d/ is no longer available. However,
|
||||
# it rc-local.service (part of systemd) still runs. We should
|
||||
# probably convert this all to a systemd job; but for now...
|
||||
mkdir -p /etc/init.d
|
||||
rclocal=/etc/init.d/boot.local
|
||||
elif [[ "${DISTRO_NAME}" =~ "gentoo" ]]; then
|
||||
rclocal=/etc/local.d/unbound.start
|
||||
mkdir -p /etc/local.d
|
||||
else
|
||||
# You'd think rc.local would be simple ...
|
||||
#
|
||||
# On Redhat systems, systemd's rc-local service looks for an
|
||||
# executable /etc/rc.d/rc.local file to run. On Debian/Ubuntu, the
|
||||
# eqivalent file is /etc/rc.local, which is missing on Debian stretch.
|
||||
#
|
||||
# Centos' systemd package symlinks /etc/rc.local to /etc/rc.d/rc.local
|
||||
# correctly. Fedora, however, does not come with an rc.local file at
|
||||
# all. Thus if we have a rc.d directory, but no rc.local file, we
|
||||
# need to create it (if you don't have an rc.d directory, and don't
|
||||
# have /etc/rc.local, then it's not clear what platform you are on).
|
||||
#
|
||||
# Bug [1] is filed to bring Fedora in-line with Centos, and has more
|
||||
# details on all this. As at 2016-10-18 is unresolved.
|
||||
#
|
||||
# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1386052
|
||||
if [[ ! -e /etc/rc.local ]] && [[ ! -e /etc/debian_version ]]; then
|
||||
if [[ ! -d /etc/rc.d ]]; then
|
||||
echo "No rc.local and no rc.d directory! See comments in 89-boot-settings"
|
||||
exit 1
|
||||
fi
|
||||
touch /etc/rc.d/rc.local
|
||||
ln -sf /etc/rc.d/rc.local /etc/rc.local
|
||||
# permissions added below. selinux context will be fixed up at
|
||||
# end of build.
|
||||
fi
|
||||
rclocal=/etc/rc.local
|
||||
fi
|
||||
|
||||
# Overwrite /etc/resolv.conf at boot (let's hope nothing else is using
|
||||
# rc.local...)
|
||||
cat >$rclocal <<EOF
|
||||
#!/bin/bash
|
||||
set -o xtrace
|
||||
|
||||
# Some providers inject dynamic network config statically. Work around this
|
||||
# for DNS nameservers. This is expected to fail on some nodes so remove -e.
|
||||
set +e
|
||||
sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-*
|
||||
sed -i -e 's/^NETCONFIG_DNS_POLICY=.*/NETCONFIG_DNS_POLICY=""/g' /etc/sysconfig/network/config
|
||||
set -e
|
||||
|
||||
echo 'nameserver 127.0.0.1' > /etc/resolv.conf
|
||||
|
||||
EOF
|
||||
|
||||
# Configure NetworkManager to not manage /etc/resolv.conf
|
||||
if [ -d /etc/NetworkManager/conf.d ]; then
|
||||
cat <<EOF |python3
|
||||
import configparser
|
||||
p = configparser.ConfigParser()
|
||||
p['main'] = {'rc-manager': 'unmanaged'}
|
||||
with open('/etc/NetworkManager/conf.d/rc-manager.conf', 'w') as fh:
|
||||
p.write(fh)
|
||||
EOF
|
||||
fi
|
||||
|
||||
# openSUSE doesn't have an iptables service, so apply the iptables rules at boot
|
||||
if [[ "${DISTRO_NAME}" =~ (opensuse) ]] ; then
|
||||
cat >>$rclocal <<EOF
|
||||
iptables-restore /etc/sysconfig/iptables
|
||||
ip6tables-restore /etc/sysconfig/ip6tables
|
||||
|
||||
EOF
|
||||
fi
|
||||
cat >>$rclocal <<EOF
|
||||
exit 0
|
||||
EOF
|
||||
|
||||
# Debian/Ubuntu ship their dummy rc.local with +x permissions, but
|
||||
# CentOS (and the file we created for Fedora) do not ... this is to
|
||||
# avoid it holding up the boot as it relies on the network being up).
|
||||
# Add +x so it runs.
|
||||
chmod a+x $rclocal
|
||||
|
||||
echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf
|
||||
|
||||
# Disable dlv. Per the unbound.conf manpage this should not be used
|
||||
# anymore but is in use by some of our distros. The problem here is it
|
||||
# does lookasides for DNSSEC which increases the number of queries and
|
||||
# introduces more points of lookup failure. Disable it to avoid these
|
||||
# problems.
|
||||
sed -i -e 's/dlv-anchor-file:/#dlv-anchor-file:/g' /etc/unbound/unbound.conf
|
||||
|
||||
# Tripleo uses dhcp
|
||||
dhcp_file=''
|
||||
if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then
|
||||
dhcp_file=/etc/dhcp/dhclient-eth0.conf
|
||||
elif [[ "$DISTRO_NAME" =~ (debian|ubuntu|gentoo) ]] ; then
|
||||
dhcp_file=/etc/dhcp/dhclient.conf
|
||||
fi
|
||||
if [ -n "$dhcp_file" ] ; then
|
||||
cat > $dhcp_file <<EOF
|
||||
option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
|
||||
send host-name "<hostname>";
|
||||
request subnet-mask, broadcast-address, routers,
|
||||
interface-mtu, rfc3442-classless-static-routes;
|
||||
supersede domain-name-servers 127.0.0.1;
|
||||
supersede domain-search "";
|
||||
supersede domain-name "";
|
||||
EOF
|
||||
fi
|
||||
|
||||
# On bionic and later, the install of the new systemd-resolved in the
|
||||
# chroot will see that there is no /etc/resolv.conf and assume it is a
|
||||
# blank system where it will be the nameserver provider. It thus
|
||||
# creates /etc/resolv.conf as a link back to its compatability files.
|
||||
# To configure systemd-resolved's resolvers you need to modify
|
||||
# /etc/systemd/resolved.conf; which would be possible, but we'd prefer
|
||||
# to be consistent across all our platforms.
|
||||
#
|
||||
# dib will copy whatever is in /etc/resolv.conf.ORIG to
|
||||
# /etc/resolv.conf as one of the final steps in image creation. Thus
|
||||
# we are hard-coding resolution to localhost (unbound) here.
|
||||
#
|
||||
# Note that with /etc/resolv.conf as a regular file, systemd-resolved
|
||||
# will also obey it for nameserver info when it starts at boot.
|
||||
rm -f /etc/resolv.conf.ORIG
|
||||
echo "nameserver 127.0.0.1" > /etc/resolv.conf.ORIG
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
upstart)
|
||||
# nothing to do
|
||||
;;
|
||||
systemd)
|
||||
systemctl enable unbound.service
|
||||
;;
|
||||
openrc)
|
||||
rc-update add unbound default
|
||||
;;
|
||||
sysv)
|
||||
# nothing to do
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported init system $DIB_INIT_SYSTEM"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@ -1,34 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=set setu setpipefail indent
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
systemd)
|
||||
glean_path_dib="/etc/systemd/system/glean@.service.d"
|
||||
mkdir -p $glean_path_dib
|
||||
nodepool_base="$(dirname $0)/../glean@.service.d"
|
||||
cp -RP $nodepool_base/override.conf $glean_path_dib/override.conf
|
||||
;;
|
||||
*)
|
||||
echo "Skipping glean systemd configuration"
|
||||
;;
|
||||
esac
|
||||
@ -1,31 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2017 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
# Journald default is to not persist logs to disk if /var/log/journal is
|
||||
# not present. Update the configuration to set storage to persistent which
|
||||
# will create /var/log/journal if necessary and store logs on disk. This
|
||||
# avoids the situation where test runs can fill the journald ring buffer
|
||||
# deleting older logs that may be important to the job.
|
||||
if [ -f /etc/systemd/journald.conf ] ; then
|
||||
sed -i -e 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf
|
||||
fi
|
||||
@ -1,28 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
# Nodepool expects this dir to exist when it boots slaves.
|
||||
# Nodepool writes environment info to this dir. We set the mode
|
||||
# to 0777 so that any user can access this env info.
|
||||
mkdir /etc/nodepool
|
||||
chmod 0777 /etc/nodepool
|
||||
@ -1,2 +0,0 @@
|
||||
[Service]
|
||||
Environment="ARGS=--interface %I --skip-dns"
|
||||
@ -1,24 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
echo $DIB_IMAGE_NAME > /etc/image-hostname.txt
|
||||
@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# Put a timestamp in the image file of the date the image was built.
|
||||
# This is echoed into the logs on each run for easy cross-reference
|
||||
|
||||
date --utc "+%Y-%m-%d %H:%M" > /etc/dib-builddate.txt
|
||||
@ -1,92 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
if [[ "$DISTRO_NAME" =~ (debian|ubuntu) ]] ; then
|
||||
rules_dir=/etc/iptables
|
||||
ipv4_rules=${rules_dir}/rules.v4
|
||||
ipv6_rules=${rules_dir}/rules.v6
|
||||
elif [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then
|
||||
rules_dir=/etc/sysconfig
|
||||
ipv4_rules=${rules_dir}/iptables
|
||||
ipv6_rules=${rules_dir}/ip6tables
|
||||
elif [[ "$DISTRO_NAME" =~ 'opensuse' ]] ; then
|
||||
rules_dir=/etc/sysconfig
|
||||
ipv4_rules=${rules_dir}/iptables
|
||||
ipv6_rules=${rules_dir}/ip6tables
|
||||
elif [[ "$DISTRO_NAME" =~ 'gentoo' ]] ; then
|
||||
rules_dir=/var/lib/iptables # not needed, part of the package install
|
||||
ipv4_rules=/var/lib/iptables/rules-save
|
||||
ipv6_rules=/var/lib/ip6tables/rules-save
|
||||
else
|
||||
echo "Unsupported operating system $DISTRO_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p $rules_dir
|
||||
|
||||
cat > $ipv4_rules << EOF
|
||||
*filter
|
||||
:INPUT ACCEPT [0:0]
|
||||
:FORWARD ACCEPT [0:0]
|
||||
:OUTPUT ACCEPT [0:0]
|
||||
:openstack-INPUT - [0:0]
|
||||
-A INPUT -j openstack-INPUT
|
||||
-A openstack-INPUT -i lo -j ACCEPT
|
||||
-A openstack-INPUT -p icmp --icmp-type any -j ACCEPT
|
||||
#-A openstack-INPUT -p udp --dport 5353 -d 224.0.0.251 -j ACCEPT
|
||||
# SSH from anywhere without -m state to avoid hanging connections on iptables-restore
|
||||
-A openstack-INPUT -m tcp -p tcp --dport 22 -j ACCEPT
|
||||
-A openstack-INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
|
||||
# Public TCP ports
|
||||
-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT
|
||||
# Ports 69 and 6385 allow to allow ironic VM nodes to reach tftp and
|
||||
# the ironic API from the neutron public net
|
||||
-A openstack-INPUT -s 172.24.4.0/23 -p udp -m udp --dport 69 -j ACCEPT
|
||||
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 6385 -j ACCEPT
|
||||
# Ports 80, 8000, 8003, 8004 from the devstack neutron public net to allow
|
||||
# nova servers to reach heat-api-cfn, heat-api-cloudwatch, heat-api
|
||||
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 80 -j ACCEPT
|
||||
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8000 -j ACCEPT
|
||||
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8003 -j ACCEPT
|
||||
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8004 -j ACCEPT
|
||||
-A openstack-INPUT -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: "
|
||||
-A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited
|
||||
COMMIT
|
||||
EOF
|
||||
|
||||
cat > $ipv6_rules << EOF
|
||||
*filter
|
||||
:INPUT ACCEPT [0:0]
|
||||
:FORWARD ACCEPT [0:0]
|
||||
:OUTPUT ACCEPT [0:0]
|
||||
:openstack-INPUT - [0:0]
|
||||
-A INPUT -j openstack-INPUT
|
||||
-A openstack-INPUT -i lo -j ACCEPT
|
||||
-A openstack-INPUT -p ipv6-icmp -j ACCEPT
|
||||
# SSH from anywhere without -m state to avoid hanging connections on iptables-restore
|
||||
-A openstack-INPUT -m tcp -p tcp --dport 22 -j ACCEPT
|
||||
-A openstack-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
|
||||
# Public TCP ports
|
||||
-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT
|
||||
-A openstack-INPUT -j REJECT --reject-with icmp6-adm-prohibited
|
||||
COMMIT
|
||||
EOF
|
||||
@ -1,28 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
# Make all cloud-init data sources match rackspace- only attempt to look
|
||||
# at ConfigDrive, not at metadata service. This is not needed if there
|
||||
# is no cloud-init
|
||||
if [ -d /etc/cloud/cloud.cfg.d ] ; then
|
||||
dd of=/etc/cloud/cloud.cfg.d/95_real_datasources.cfg <<EOF
|
||||
datasource_list: [ ConfigDrive, None ]
|
||||
EOF
|
||||
fi
|
||||
@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
python3 -m venv /usr/os-testr-env
|
||||
if [[ ${DIB_RELEASE} == 'xenial' ]]; then
|
||||
# The pip on xenial can't figure out it shouldn't install
|
||||
# the latest pip; this is the last to support 3.5
|
||||
/usr/os-testr-env/bin/pip install --upgrade pip==20.3.4
|
||||
else
|
||||
/usr/os-testr-env/bin/pip install --upgrade pip
|
||||
fi
|
||||
/usr/os-testr-env/bin/pip install os-testr
|
||||
@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Remove cron jobs
|
||||
# We create fresh servers for these hosts, and they are used once. They don't
|
||||
# need to do things like update the locatedb or the mandb or rotate logs
|
||||
# or really any of those things. We only want code running here that we want
|
||||
# here.
|
||||
|
||||
# dib-lint: disable=sete setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
rm -f /etc/cron.{monthly,weekly,daily,hourly,d}/*
|
||||
@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=set setu setpipefail indent
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
# This will disable the disable Privacy extensions for IPv6 (RFC3041)
|
||||
cat > /etc/sysctl.d/99-cloudimg-ipv6.conf <<EOF
|
||||
# See https://bugs.launchpad.net/ubuntu/+source/procps/+bug/1068756
|
||||
net.ipv6.conf.all.use_tempaddr=0
|
||||
net.ipv6.conf.default.use_tempaddr=0
|
||||
EOF
|
||||
@ -1 +0,0 @@
|
||||
unbound:
|
||||
@ -1,14 +0,0 @@
|
||||
{
|
||||
"release": {
|
||||
"debian": {
|
||||
"bookworm": {
|
||||
"unbound": "unbound dns-root-data"
|
||||
}
|
||||
},
|
||||
"ubuntu": {
|
||||
"noble": {
|
||||
"unbound": "unbound dns-root-data"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
if [[ "$DISTRO_NAME" =~ (debian|ubuntu) ]] ; then
|
||||
service_name=netfilter-persistent
|
||||
elif [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then
|
||||
service_name=iptables
|
||||
elif [[ "$DISTRO_NAME" == 'opensuse' ]] ; then
|
||||
# handle iptables through init.d instead of a service
|
||||
exit 0
|
||||
elif [[ "$DISTRO_NAME" == 'gentoo' ]] ; then
|
||||
# for use in loops in both openrc and systemd
|
||||
service_name='iptables ip6tables'
|
||||
else
|
||||
echo "Unsupported operating system $DISTRO_NAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$DIB_INIT_SYSTEM" in
|
||||
upstart)
|
||||
# nothing to do
|
||||
;;
|
||||
systemd)
|
||||
if [[ "$DISTRO_NAME" == 'gentoo' ]] ; then
|
||||
for service in ${service_name}; do
|
||||
systemctl enable "${service}-restore.service"
|
||||
systemctl enable "${service}-store.service"
|
||||
done
|
||||
else
|
||||
systemctl enable ${service_name}.service
|
||||
fi
|
||||
if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then
|
||||
systemctl enable ip6tables.service
|
||||
fi
|
||||
;;
|
||||
openrc)
|
||||
for service in ${service_name}; do
|
||||
rc-update add "${service}" default
|
||||
done
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported init system $DIB_INIT_SYSTEM"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
# Copyright 2016 Red Hat, Inc.
|
||||
# Copyright 2018 SUSE Linux, LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
mkdir -p /usr/local/bindep-fallback
|
||||
cp -a /opt/git/opendev.org/openstack/project-config/nodepool/elements/bindep-fallback.txt \
|
||||
/usr/local/bindep-fallback/
|
||||
@ -1,43 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# workaround a broken maintainer script in iptables-persistent that
|
||||
# assumes it can execute things during postinst. Since we're installing
|
||||
# in a chroot, we're actually not running the right kernel for its assumptions
|
||||
# to work.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
# If lsb_release is missing, just do nothing.
|
||||
DISTRO=$(lsb_release -si) || true
|
||||
|
||||
case $DISTRO in
|
||||
'Ubuntu'|'Debian')
|
||||
PRESEED=$(mktemp)
|
||||
cat > $PRESEED <<EOF
|
||||
d-i iptables-persistent/autosave_done boolean true
|
||||
d-i iptables-persistent/autosave_v4 boolean false
|
||||
d-i iptables-persistent/autosave_v6 boolean false
|
||||
EOF
|
||||
debconf-set-selections $PRESEED
|
||||
rm $PRESEED
|
||||
;;
|
||||
esac
|
||||
@ -1,30 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2017 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# NOTE(pabelanger): Because our AFS mirrors are not signed, we need to allow
|
||||
# unauthenticated packages and insecure repos.
|
||||
if [ -d "$TARGET_ROOT/etc/apt/apt.conf.d" ]; then
|
||||
echo "APT::Get::AllowUnauthenticated \"true\";" | sudo tee $TARGET_ROOT/etc/apt/apt.conf.d/95allow-unauthenticated
|
||||
echo "Acquire::AllowInsecureRepositories \"true\";" | sudo tee -a $TARGET_ROOT/etc/apt/apt.conf.d/95allow-unauthenticated
|
||||
fi
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2017 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# NOTE(frickler): Our images builds always install the newest updates,
|
||||
# so we need this in order to avoid conflicts
|
||||
if [[ ":jammy: :noble:" =~ :${DIB_RELEASE}: ]]; then
|
||||
if [ -d "$TARGET_ROOT/etc/apt/apt.conf.d" ]; then
|
||||
echo "APT::Get::Always-Include-Phased-Updates \"true\";" | sudo tee $TARGET_ROOT/etc/apt/apt.conf.d/95phased-updates
|
||||
fi
|
||||
fi
|
||||
@ -1,19 +0,0 @@
|
||||
===============
|
||||
openstack-repos
|
||||
===============
|
||||
|
||||
Download all repos and packages that might be needed.
|
||||
|
||||
Environment variables:
|
||||
----------------------
|
||||
|
||||
DIB_CUSTOM_PROJECTS_LIST_URL
|
||||
:Required: No
|
||||
:Default: None
|
||||
:Description: Url to a yaml file contains custom list of repos.
|
||||
The custom yaml file has the same structure as the default file:
|
||||
'https://opendev.org/openstack/project-config/raw/gerrit/projects.yaml'
|
||||
Download only the repos that appear in the custom file rather than
|
||||
downloading all openstack repos that appear in the default file.
|
||||
:Example:
|
||||
DIB_CUSTOM_PROJECTS_LIST_URL='file:///etc//project-config//gerrit//custom_projects.yaml'
|
||||
@ -1,2 +0,0 @@
|
||||
cache-url
|
||||
source-repositories
|
||||
@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) 2011-2013 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import yaml
|
||||
from urllib.request import urlopen
|
||||
from urllib.request import URLError
|
||||
|
||||
URL = ('https://opendev.org/openstack/project-config/'
|
||||
'raw/gerrit/projects.yaml')
|
||||
|
||||
TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH']
|
||||
PROJECTS_REPOS = os.path.join(TMP_HOOKS_PATH,
|
||||
'source-repository-projects-yaml')
|
||||
GIT_BASE = os.environ.get('GIT_BASE', 'https://opendev.org')
|
||||
|
||||
CUSTOM_PROJECTS_LIST_URL = os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
|
||||
|
||||
|
||||
def get_project_list(url):
|
||||
try:
|
||||
projects = []
|
||||
for f in yaml.safe_load(urlopen(url)):
|
||||
# Skip repos that are inactive
|
||||
project = f['project']
|
||||
dirname = os.path.dirname(project)
|
||||
if 'attic' in dirname or dirname == 'stackforge':
|
||||
continue
|
||||
acl = f.get('acl-config')
|
||||
# Ignore retired repositories
|
||||
if acl and os.path.basename(acl) == 'retired.config':
|
||||
continue
|
||||
projects.append(project)
|
||||
|
||||
return projects
|
||||
except URLError:
|
||||
print("Could not open project list url: '%s'" % url)
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
projects = []
|
||||
if CUSTOM_PROJECTS_LIST_URL:
|
||||
projects = get_project_list(CUSTOM_PROJECTS_LIST_URL)
|
||||
|
||||
if not projects:
|
||||
projects = get_project_list(URL)
|
||||
|
||||
with open(PROJECTS_REPOS, 'w') as projects_list:
|
||||
for project in projects:
|
||||
args = dict(
|
||||
name=os.path.basename(project),
|
||||
location=os.path.join('/opt/git/opendev.org', project),
|
||||
url='%s/%s.git' % (GIT_BASE, project),
|
||||
ref='*')
|
||||
|
||||
projects_list.write("%(name)s git %(location)s "
|
||||
"%(url)s %(ref)s\n" % args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,25 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# dib-lint: disable=setu setpipefail
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -e
|
||||
|
||||
if [[ "$DISTRO_NAME" =~ (centos|rhel7) ]] ; then
|
||||
yum -y install --enablerepo=epel haveged
|
||||
fi
|
||||
@ -1,17 +0,0 @@
|
||||
zuul-worker
|
||||
===========
|
||||
|
||||
Setup a node to be a zuul worker
|
||||
|
||||
User Creation
|
||||
=============
|
||||
|
||||
This element bakes in a ``zuul`` user on the host for the zuul-worker
|
||||
process to log in with.
|
||||
|
||||
By default login permissions (``authorized_keys``) will be populated
|
||||
for the ``zuul`` user from ``~/.ssh/id_rsa.pub`` -- i.e. the public
|
||||
key of the currently building user. Specify an alternative filename
|
||||
in ``ZUUL_USER_SSH_PUBLIC_KEY`` to override this.
|
||||
|
||||
The ``zuul`` user is provided with passwordless ``sudo`` access.
|
||||
@ -1 +0,0 @@
|
||||
package-installs
|
||||
@ -1,19 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
ZUUL_USER_SSH_PUBLIC_KEY=${ZUUL_USER_SSH_PUBLIC_KEY:-$HOME/.ssh/id_rsa.pub}
|
||||
|
||||
if [ -f "$ZUUL_USER_SSH_PUBLIC_KEY" ]; then
|
||||
# save the public key inside the chroot from specific file
|
||||
cat $ZUUL_USER_SSH_PUBLIC_KEY >> $TMP_HOOKS_PATH/zuul-user-ssh-public-key
|
||||
elif [ -n "$ZUUL_USER_SSH_PUBLIC_KEY" ]; then
|
||||
# save the specific public key inside the chroot from env derectly
|
||||
echo "$ZUUL_USER_SSH_PUBLIC_KEY" >> $TMP_HOOKS_PATH/zuul-user-ssh-public-key
|
||||
else
|
||||
die "Can not find public key for zuul user!"
|
||||
fi
|
||||
@ -1,60 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# Add zuul user and group. Note we don't want to rely on
|
||||
# "useradd"'s group adding behaviour, because it might differ across
|
||||
# distros.
|
||||
groupadd zuul
|
||||
useradd -m zuul -g zuul -s /bin/bash
|
||||
|
||||
cat > /etc/sudoers.d/zuul << EOF
|
||||
zuul ALL=(ALL) NOPASSWD:ALL
|
||||
EOF
|
||||
chmod 0440 /etc/sudoers.d/zuul
|
||||
|
||||
visudo -c || die "Error setting zuul sudo!"
|
||||
|
||||
# this was copied from outside the chroot by extras.d
|
||||
_pub_key=/tmp/in_target.d/zuul-user-ssh-public-key
|
||||
if [ ! -f $_pub_key ]; then
|
||||
die "Can not find Zuul public key!"
|
||||
fi
|
||||
|
||||
mkdir -p /home/zuul/.ssh
|
||||
chmod 700 /home/zuul/.ssh
|
||||
cp $_pub_key /home/zuul/.ssh/authorized_keys
|
||||
|
||||
# cleanup everything to the right owner
|
||||
chown -R zuul:zuul /home/zuul
|
||||
|
||||
# a lot of caching happens in extra-data.d (for "historical" reasons).
|
||||
# We've put the cache stuff into /opt/cache/files, but again, for
|
||||
# "historical" reasons, ensure this is available in /home/zuul
|
||||
#
|
||||
# We do this for zuul as relative paths to the current user's homedir
|
||||
# are used in places like devstack.
|
||||
#
|
||||
# Check if the cache exists as we don't have a strict dependency on the
|
||||
# devstack-cache element. This allows you to build an image without
|
||||
# incurring the cost of caching all the things.
|
||||
if [ -d /opt/cache/files ] ; then
|
||||
mkdir -p /home/zuul/cache
|
||||
chown zuul:zuul /home/zuul/cache
|
||||
ln -sf /opt/cache/files /home/zuul/cache/files
|
||||
# but make sure the cache is readable by everyone
|
||||
chmod -R a+rX /opt/cache/files/*
|
||||
fi
|
||||
|
||||
# New versions of git don't let you clone repos as a different user
|
||||
# than the user owning the repo by default for security reasons.
|
||||
# As above we cache git repos during extra-data.d in /opt/git/ and they
|
||||
# end up owned by root. Chown them to zuul here to avoid permissions
|
||||
# issues with the most likely user to interact with the git cache( zuul).
|
||||
if [ -d /opt/git ] ; then
|
||||
chown -R zuul:zuul /opt/git
|
||||
fi
|
||||
@ -1,27 +0,0 @@
|
||||
- name: Compress image
|
||||
when: "upload_image_format in ['raw', 'vhd']"
|
||||
command: zstd '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_format }}'
|
||||
- name: Set extension
|
||||
when: "upload_image_format in ['raw', 'vhd']"
|
||||
set_fact:
|
||||
upload_image_extension: '{{ upload_image_format }}.zst'
|
||||
- name: Set extension
|
||||
when: "upload_image_format not in ['raw', 'vhd']"
|
||||
set_fact:
|
||||
upload_image_extension: '{{ upload_image_format }}'
|
||||
- name: Upload image
|
||||
no_log: true
|
||||
include_role:
|
||||
name: image-upload-swift
|
||||
vars:
|
||||
cloud:
|
||||
auth_type: 'v3applicationcredential'
|
||||
auth:
|
||||
auth_url: 'https://keystone.api.sjc3.rackspacecloud.com/v3'
|
||||
application_credential_id: '{{ image_upload_secret.application_credential_id }}'
|
||||
application_credential_secret: '{{ image_upload_secret.application_credential_secret }}'
|
||||
user_domain_name: rackspace_cloud_domain
|
||||
container: images-1f49951f5beb
|
||||
filename: '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_extension }}'
|
||||
name: '{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_extension }}'
|
||||
delete_after: 259200
|
||||
@ -1,8 +0,0 @@
|
||||
- hosts: all
|
||||
tasks:
|
||||
- name: Upload image
|
||||
when: image_upload_secret is defined and zuul_success
|
||||
with_items: '{{ build_diskimage_formats }}'
|
||||
loop_control:
|
||||
loop_var: upload_image_format
|
||||
include_tasks: post-inner.yaml
|
||||
@ -1,38 +0,0 @@
|
||||
- hosts: all
|
||||
pre_tasks:
|
||||
- name: Install custom element requirements
|
||||
become: true
|
||||
package:
|
||||
state: present
|
||||
name:
|
||||
- python3-yaml
|
||||
# The following are required for the image upload
|
||||
- python3-openstacksdk
|
||||
- python3-oslo.utils
|
||||
- zstd
|
||||
- name: Setup dib directories
|
||||
become: true
|
||||
shell: |
|
||||
set -x
|
||||
|
||||
mkdir -p /opt/dib_cache
|
||||
chown {{ ansible_user }} /opt/dib_cache
|
||||
chgrp {{ ansible_user }} /opt/dib_cache
|
||||
|
||||
mkdir -p /opt/dib_tmp
|
||||
|
||||
if [ -b /dev/vdb ]; then
|
||||
DEV='/dev/vdb'
|
||||
elif [ -b /dev/xvde ]; then
|
||||
DEV='/dev/xvde'
|
||||
fi
|
||||
|
||||
if [ -n "$DEV" ]; then
|
||||
mkfs.ext4 $DEV
|
||||
mount $DEV /opt/dib_tmp
|
||||
fi
|
||||
|
||||
chown {{ ansible_user }} /opt/dib_tmp
|
||||
chgrp {{ ansible_user }} /opt/dib_tmp
|
||||
roles:
|
||||
- make-source-repositories-cache
|
||||
@ -1,168 +0,0 @@
|
||||
# Copyright 2014 Rackspace Australia
|
||||
# Copyright 2018 Red Hat, Inc
|
||||
# Copyright 2024 Acme Gating, LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import openstack
|
||||
import requests.exceptions
|
||||
import keystoneauth1.exceptions
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
|
||||
SEGMENT_SIZE = 500000000 # 500MB
|
||||
|
||||
|
||||
def get_cloud(cloud):
|
||||
if isinstance(cloud, dict):
|
||||
config = openstack.config.loader.OpenStackConfig().get_one(**cloud)
|
||||
return openstack.connection.Connection(
|
||||
config=config,
|
||||
pool_executor=concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=10
|
||||
))
|
||||
else:
|
||||
return openstack.connect(cloud=cloud)
|
||||
|
||||
|
||||
def _add_etag_to_manifest(self, *args, **kw):
|
||||
return
|
||||
|
||||
|
||||
def prune(cloud, container, delete_after):
|
||||
# In case the automatic expiration doesn't work, manually prune old uploads
|
||||
if not delete_after:
|
||||
return
|
||||
target = (datetime.datetime.now(datetime.UTC) -
|
||||
datetime.timedelta(seconds=delete_after))
|
||||
endpoint = cloud.object_store.get_endpoint()
|
||||
url = os.path.join(endpoint, container)
|
||||
for obj in cloud.object_store.objects(container):
|
||||
ts = datetime.datetime.fromisoformat(obj['last_modified'])
|
||||
ts = ts.replace(tzinfo=datetime.UTC)
|
||||
if ts < target:
|
||||
path = os.path.join(url, obj.name)
|
||||
try:
|
||||
cloud.session.delete(path)
|
||||
except keystoneauth1.exceptions.http.NotFound:
|
||||
pass
|
||||
|
||||
|
||||
def run(cloud, container, filename, name, delete_after=None):
|
||||
# Monkey-patch sdk so that the SLO upload does not add the etag;
|
||||
# this works around an issue with rackspace-flex.
|
||||
cloud.object_store._add_etag_to_manifest = _add_etag_to_manifest
|
||||
prune(cloud, container, delete_after)
|
||||
headers = {}
|
||||
if delete_after:
|
||||
headers['X-Delete-After'] = str(delete_after)
|
||||
endpoint = cloud.object_store.get_endpoint()
|
||||
cloud.object_store.create_object(
|
||||
container,
|
||||
name=name,
|
||||
filename=filename,
|
||||
segment_size=SEGMENT_SIZE,
|
||||
**headers)
|
||||
url = os.path.join(endpoint, container, name)
|
||||
return url
|
||||
|
||||
|
||||
def ansible_main():
|
||||
module = AnsibleModule(
|
||||
argument_spec=dict(
|
||||
cloud=dict(required=True, type='raw'),
|
||||
container=dict(required=True, type='str'),
|
||||
filename=dict(required=True, type='path'),
|
||||
name=dict(required=True, type='str'),
|
||||
delete_after=dict(type='int'),
|
||||
)
|
||||
)
|
||||
|
||||
p = module.params
|
||||
cloud = get_cloud(p.get('cloud'))
|
||||
try:
|
||||
url = run(
|
||||
cloud,
|
||||
p.get('container'),
|
||||
p.get('filename'),
|
||||
p.get('name'),
|
||||
delete_after=p.get('delete_after'),
|
||||
)
|
||||
except (keystoneauth1.exceptions.http.HttpError,
|
||||
requests.exceptions.RequestException):
|
||||
s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name)
|
||||
s += "\n" + traceback.format_exc()
|
||||
module.fail_json(
|
||||
changed=False,
|
||||
msg=s,
|
||||
cloud=cloud.name,
|
||||
region_name=cloud.config.region_name)
|
||||
module.exit_json(
|
||||
changed=True,
|
||||
url=url,
|
||||
)
|
||||
|
||||
|
||||
def cli_main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upload image to swift"
|
||||
)
|
||||
parser.add_argument('--verbose', action='store_true',
|
||||
help='show debug information')
|
||||
parser.add_argument('cloud',
|
||||
help='Name of the cloud to use when uploading')
|
||||
parser.add_argument('container',
|
||||
help='Name of the container to use when uploading')
|
||||
parser.add_argument('filename',
|
||||
help='the file to upload')
|
||||
parser.add_argument('name',
|
||||
help='the object name')
|
||||
parser.add_argument('--delete-after',
|
||||
help='Number of seconds to delete object after '
|
||||
'upload. Default is 3 days (259200 seconds) '
|
||||
'and if set to 0 X-Delete-After will not be set',
|
||||
type=int)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
# Set requests log level accordingly
|
||||
logging.getLogger("requests").setLevel(logging.DEBUG)
|
||||
logging.getLogger("keystoneauth").setLevel(logging.INFO)
|
||||
logging.getLogger("stevedore").setLevel(logging.INFO)
|
||||
logging.captureWarnings(True)
|
||||
|
||||
url = run(
|
||||
get_cloud(args.cloud),
|
||||
args.container,
|
||||
args.filename,
|
||||
args.name,
|
||||
delete_after=args.delete_after,
|
||||
)
|
||||
print(url)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not sys.stdin.isatty():
|
||||
ansible_main()
|
||||
else:
|
||||
cli_main()
|
||||
@ -1,56 +0,0 @@
|
||||
# Run the checksums in the background while we're uploading
|
||||
- name: Get sha256 hash
|
||||
stat:
|
||||
path: '{{ filename }}'
|
||||
checksum_algorithm: sha256
|
||||
async: 600
|
||||
poll: 0
|
||||
register: sha256_task
|
||||
|
||||
- name: Get md5 hash
|
||||
stat:
|
||||
path: '{{ filename }}'
|
||||
checksum_algorithm: md5
|
||||
async: 600
|
||||
poll: 0
|
||||
register: md5_task
|
||||
|
||||
- name: Upload image to swift
|
||||
no_log: true
|
||||
image_upload_swift:
|
||||
cloud: '{{ cloud }}'
|
||||
container: '{{ container }}'
|
||||
filename: '{{ filename }}'
|
||||
name: '{{ name }}'
|
||||
delete_after: '{{ delete_after }}'
|
||||
register: upload_results
|
||||
|
||||
- name: Wait for sha256
|
||||
async_status:
|
||||
jid: "{{ sha256_task.ansible_job_id }}"
|
||||
register: sha256
|
||||
until: sha256.finished
|
||||
retries: 1
|
||||
delay: 10
|
||||
|
||||
- name: Wait for md5
|
||||
async_status:
|
||||
jid: "{{ md5_task.ansible_job_id }}"
|
||||
register: md5
|
||||
until: md5.finished
|
||||
retries: 1
|
||||
delay: 10
|
||||
|
||||
- name: Return artifact to Zuul
|
||||
zuul_return:
|
||||
data:
|
||||
zuul:
|
||||
artifacts:
|
||||
- name: '{{ upload_image_format }} image'
|
||||
url: '{{ upload_results.url }}'
|
||||
metadata:
|
||||
type: 'zuul_image'
|
||||
image_name: '{{ build_diskimage_image_name }}'
|
||||
format: '{{ upload_image_format }}'
|
||||
sha256: '{{ sha256.stat.checksum }}'
|
||||
md5sum: '{{ md5.stat.checksum }}'
|
||||
@ -1,12 +0,0 @@
|
||||
Reformat the git repo cache for source-respositories
|
||||
|
||||
This reverses the process of creating the /opt/git cache on worker
|
||||
nodes. It turns that cache into the format used by the
|
||||
source-repositories diskimage builder element.
|
||||
|
||||
**Role Variables**
|
||||
|
||||
.. zuul:rolevar:: cached_repos_root
|
||||
:default: /opt/git
|
||||
|
||||
The root of the cached repos.
|
||||
@ -1,2 +0,0 @@
|
||||
# Matches prepare_workspace_root
|
||||
cached_repos_root: /opt/git
|
||||
@ -1,27 +0,0 @@
|
||||
- name: Reformat the git repo cache for source-repositories
|
||||
environment: "{{ build_diskimage_environment }}"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
shell: |
|
||||
GIT_CACHE_ROOT="{{ cached_repos_root }}"
|
||||
|
||||
REPOTYPE=git
|
||||
CACHE_BASE=${DIB_IMAGE_CACHE}/source-repositories
|
||||
mkdir -p ${CACHE_BASE}
|
||||
|
||||
# Note: maxdepth 4 is based on the current opendev practice of
|
||||
# not having repos deeper than two directory levels. This is
|
||||
# fragile and should probably be replaced with something better,
|
||||
# but as written it saves considerable time.
|
||||
for NODE_CACHE_PATH in $(find $GIT_CACHE_ROOT -maxdepth 4 -type d -name .git); do
|
||||
REPOPATH=$(dirname $NODE_CACHE_PATH)
|
||||
REPONAME=$(basename $REPOPATH)
|
||||
REPOLOCATION="$(echo $REPOPATH | sed "s,$GIT_CACHE_ROOT/,https://,").git"
|
||||
|
||||
# From diskimage-builder element source-repositories
|
||||
CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
|
||||
CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME}
|
||||
CACHE_PATH=${CACHE_BASE}/${CACHE_NAME}
|
||||
|
||||
mv ${REPOPATH} ${CACHE_PATH}
|
||||
done
|
||||
Loading…
x
Reference in New Issue
Block a user