Removed fuel_agent and fuel_agent_ci
We've removed fuel_agent into a separate repository, so we don't need this directory here any more. Closes-Bug: #1471849 Change-Id: I165259020721a4f6a87dce3a419bfdf64a5d0fcb
This commit is contained in:
parent
bb362555db
commit
ea8a7f94dd
|
@ -1,5 +0,0 @@
|
|||
[DEFAULT]
|
||||
test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -s fuel_agent/tests -p "*.py" $LISTOPT $IDOPTION
|
||||
test_id_option=--load-list $IDFILE
|
||||
test_list_option=--list
|
||||
test_run_concurrency=echo 1
|
|
@ -1,105 +0,0 @@
|
|||
#cloud-boothook
|
||||
#!/bin/bash
|
||||
|
||||
function add_str_to_file_if_not_exists {
|
||||
file=$1
|
||||
str=$2
|
||||
val=$3
|
||||
if ! grep -q "^ *${str}" $file; then
|
||||
echo $val >> $file
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
cloud-init-per instance disable_selinux_on_the_fly setenforce 0
|
||||
|
||||
cloud-init-per instance disable_selinux sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/sysconfig/selinux
|
||||
|
||||
|
||||
# configure udev rules
|
||||
|
||||
# udev persistent net
|
||||
cloud-init-per instance udev_persistent_net1 service network stop
|
||||
|
||||
ADMIN_MAC={{ common.admin_mac }}
|
||||
ADMIN_IF=$(echo {{ common.udevrules }} | sed 's/[,=]/\n/g' | grep "$ADMIN_MAC" | cut -d_ -f2 | head -1)
|
||||
cloud-init-per instance configure_admin_interface /bin/sh -c "echo -e \"# FROM COBBLER SNIPPET\nDEVICE=$ADMIN_IF\nIPADDR={{ common.admin_ip }}\nNETMASK={{ common.admin_mask }}\nBOOTPROTO=none\nONBOOT=yes\nUSERCTL=no\n\" | tee /etc/sysconfig/network-scripts/ifcfg-$ADMIN_IF"
|
||||
|
||||
cloud-init-per instance set_gateway /bin/sh -c 'echo GATEWAY="{{ common.gw }}" | tee -a /etc/sysconfig/network'
|
||||
|
||||
cloud-init-per instance udev_persistent_net5 service network start
|
||||
|
||||
# end of udev
|
||||
|
||||
#FIXME(agordeev): if operator updates dns settings on masternode after the node had been provisioned,
|
||||
# cloud-init will start to generate resolv.conf with non-actual data
|
||||
cloud-init-per instance resolv_conf_remove rm -f /etc/resolv.conf
|
||||
cloud-init-per instance resolv_conf_header /bin/sh -c 'echo "# re-generated by cloud-init boothook only at the first boot;" | tee /etc/resolv.conf'
|
||||
cloud-init-per instance resolv_conf_search /bin/sh -c 'echo "search {{ common.search_domain|replace('"','') }}" | tee -a /etc/resolv.conf'
|
||||
cloud-init-per instance resolv_conf_domain /bin/sh -c 'echo "domain {{ common.search_domain|replace('"','') }}" | tee -a /etc/resolv.conf'
|
||||
cloud-init-per instance resolv_conf_nameserver /bin/sh -c 'echo nameserver {{ common.master_ip }} | tee -a /etc/resolv.conf'
|
||||
|
||||
# configure black module lists
|
||||
# virt-what should be installed
|
||||
if [ ! -f /etc/modprobe.d/blacklist-i2c_piix4.conf ]; then
|
||||
([[ $(virt-what) = "virtualbox" ]] && echo "blacklist i2c_piix4" >> /etc/modprobe.d/blacklist-i2c_piix4.conf || :)
|
||||
modprobe -r i2c_piix4
|
||||
fi
|
||||
|
||||
cloud-init-per instance conntrack_ipv4 /bin/sh -c 'echo nf_conntrack_ipv4 | tee -a /etc/rc.modules'
|
||||
cloud-init-per instance conntrack_ipv6 /bin/sh -c 'echo nf_conntrack_ipv6 | tee -a /etc/rc.modules'
|
||||
cloud-init-per instance chmod_rc_modules chmod +x /etc/rc.modules
|
||||
cloud-init-per instance conntrack_max /bin/sh -c 'echo "net.nf_conntrack_max=1048576" | tee -a /etc/sysctl.conf'
|
||||
|
||||
cloud-init-per instance conntrack_ipv4_load modprobe nf_conntrack_ipv4
|
||||
cloud-init-per instance conntrack_ipv6_load modprobe nf_conntrack_ipv6
|
||||
cloud-init-per instance conntrack_max_set sysctl -w "net.nf_conntrack_max=1048576"
|
||||
|
||||
cloud-init-per instance mkdir_coredump mkdir -p /var/log/coredump
|
||||
cloud-init-per instance set_coredump /bin/sh -c 'echo -e "kernel.core_pattern=/var/log/coredump/core.%e.%p.%h.%t" | tee -a /etc/sysctl.conf'
|
||||
cloud-init-per instance set_coredump_sysctl sysctl -w "kernel.core_pattern=/var/log/coredump/core.%e.%p.%h.%t"
|
||||
cloud-init-per instance set_chmod chmod 777 /var/log/coredump
|
||||
cloud-init-per instance set_limits /bin/sh -c 'echo -e "* soft core unlimited\n* hard core unlimited" | tee -a /etc/security/limits.conf'
|
||||
|
||||
|
||||
#NOTE: disabled for centos?
|
||||
#cloud-init-per instance dhclient echo 'supersede routers 0;' | tee /etc/dhcp/dhclient.conf
|
||||
|
||||
# ntp sync
|
||||
# '| tee /dev/null' is needed for returning zero execution code always
|
||||
cloud-init-per instance stop_ntpd /bin/sh -c 'service ntpd stop | tee /dev/null'
|
||||
cloud-init-per instance sync_date ntpdate -t 4 -b {{ common.master_ip }}
|
||||
cloud-init-per instance sync_hwclock hwclock --systohc
|
||||
|
||||
cloud-init-per instance edit_ntp_conf1 sed -i '/^\s*tinker panic/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf2 sed -i '1 i tinker panic 0' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf_mkdir mkdir -p /var/lib/ntp
|
||||
cloud-init-per instance edit_ntp_conf3 /bin/sh -c 'echo 0 | tee /var/lib/ntp/drift'
|
||||
cloud-init-per instance edit_ntp_conf4 chown ntp: /var/lib/ntp/drift
|
||||
cloud-init-per instance edit_ntp_conf5 sed -i '/^\s*server/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf6 /bin/sh -c 'echo "server {{ common.master_ip }} burst iburst" | tee -a /etc/ntp.conf'
|
||||
|
||||
|
||||
# Point installed ntpd to Master node
|
||||
cloud-init-per instance set_ntpdate sed -i 's/SYNC_HWCLOCK\s*=\s*no/SYNC_HWCLOCK=yes/' /etc/sysconfig/ntpdate
|
||||
cloud-init-per instance set_ntpd_0 chkconfig ntpd on
|
||||
cloud-init-per instance set_ntpd_1 chkconfig ntpdate on
|
||||
cloud-init-per instance start_ntpd service ntpd start
|
||||
|
||||
cloud-init-per instance removeUseDNS sed -i --follow-symlinks -e '/UseDNS/d' /etc/ssh/sshd_config
|
||||
add_str_to_file_if_not_exists /etc/ssh/sshd_config 'UseDNS' 'UseDNS no'
|
||||
|
||||
cloud-init-per instance gssapi_disable sed -i -e "/^\s*GSSAPICleanupCredentials yes/d" -e "/^\s*GSSAPIAuthentication yes/d" /etc/ssh/sshd_config
|
||||
|
||||
cloud-init-per instance nailgun_agent_0 /bin/sh -c 'echo "rm -f /etc/nailgun-agent/nodiscover" | tee /etc/rc.local'
|
||||
cloud-init-per instance nailgun_agent_1 /bin/sh -c 'echo "flock -w 0 -o /var/lock/agent.lock -c \"/opt/nailgun/bin/agent >> /var/log/nailgun-agent.log 2>&1\"" | tee -a /etc/rc.local'
|
||||
|
||||
# Copying default bash settings to the root directory
|
||||
cloud-init-per instance skel_bash cp -f /etc/skel/.bash* /root/
|
||||
|
||||
# Puppet config
|
||||
cloud-init-per instance hiera_puppet mkdir -p /etc/puppet /var/lib/hiera
|
||||
cloud-init-per instance touch_puppet touch /var/lib/hiera/common.yaml /etc/puppet/hiera.yaml
|
||||
|
||||
# Mcollective enable
|
||||
cloud-init-per instance mcollective_enable sed -i /etc/rc.d/init.d/mcollective -e 's/\(# chkconfig:\s\+[-0-6]\+\) [0-9]\+ \([0-9]\+\)/\1 81 \2/'
|
|
@ -1,92 +0,0 @@
|
|||
#cloud-boothook
|
||||
#!/bin/bash
|
||||
|
||||
function add_str_to_file_if_not_exists {
|
||||
file=$1
|
||||
str=$2
|
||||
val=$3
|
||||
if ! grep -q "^ *${str}" $file; then
|
||||
echo $val >> $file
|
||||
fi
|
||||
}
|
||||
|
||||
cloud-init-per instance wipe_sources_list_templates /bin/sh -c 'echo | tee /etc/cloud/templates/sources.list.ubuntu.tmpl'
|
||||
|
||||
# configure udev rules
|
||||
|
||||
# udev persistent net
|
||||
cloud-init-per instance udev_persistent_net1 /etc/init.d/networking stop
|
||||
|
||||
ADMIN_MAC={{ common.admin_mac }}
|
||||
ADMIN_IF=$(echo {{ common.udevrules }} | sed 's/[,=]/\n/g' | grep "$ADMIN_MAC" | cut -d_ -f2 | head -1)
|
||||
# Check if we do not already have static config (or interface seems unconfigured)
|
||||
if [ ! -d "/etc/network/interfaces.d" ]; then
|
||||
mkdir -p /etc/network/interfaces.d
|
||||
echo 'source /etc/network/interfaces.d/*' > /etc/network/interfaces
|
||||
fi
|
||||
if [ ! -e "/etc/network/interfaces.d/ifcfg-$ADMIN_IF" ]; then
|
||||
echo -e "auto $ADMIN_IF\niface $ADMIN_IF inet static\n\taddress {{ common.admin_ip }}\n\tnetmask {{ common.admin_mask }}\n\tgateway {{ common.gw }}" > /etc/network/interfaces.d/ifcfg-"$ADMIN_IF"
|
||||
fi
|
||||
|
||||
cloud-init-per instance udev_persistent_net5 /etc/init.d/networking start
|
||||
|
||||
# end of udev
|
||||
|
||||
#FIXME(agordeev): if operator updates dns settings on masternode after the node had been provisioned,
|
||||
# cloud-init will start to generate resolv.conf with non-actual data
|
||||
cloud-init-per instance resolv_conf_mkdir mkdir -p /etc/resolvconf/resolv.conf.d
|
||||
cloud-init-per instance resolv_conf_remove rm -f /etc/resolv.conf
|
||||
cloud-init-per instance resolv_conf_head_remove rm -f /etc/resolvconf/resolv.conf.d/head
|
||||
cloud-init-per instance resolv_conf_header /bin/sh -c 'echo "# re-generated by cloud-init boothook only at the first boot;" | tee /etc/resolv.conf'
|
||||
cloud-init-per instance resolv_conf_search /bin/sh -c 'echo "search {{ common.search_domain|replace('"','') }}" | tee -a /etc/resolv.conf'
|
||||
cloud-init-per instance resolv_conf_domain /bin/sh -c 'echo "domain {{ common.search_domain|replace('"','') }}" | tee -a /etc/resolv.conf'
|
||||
cloud-init-per instance resolv_conf_head_header /bin/sh -c 'echo "# re-generated by cloud-init boothook only at the first boot;" | tee /etc/resolvconf/resolv.conf.d/head'
|
||||
cloud-init-per instance resolv_conf_head_search /bin/sh -c 'echo "search {{ common.search_domain|replace('"','') }}" | tee -a /etc/resolvconf/resolv.conf.d/head'
|
||||
cloud-init-per instance resolv_conf_head_domain /bin/sh -c 'echo "domain {{ common.search_domain|replace('"','') }}" | tee -a /etc/resolvconf/resolv.conf.d/head'
|
||||
cloud-init-per instance resolv_conf_nameserver /bin/sh -c 'echo nameserver {{ common.master_ip|replace('"','') }} | tee -a /etc/resolv.conf'
|
||||
cloud-init-per instance resolv_conf_head_nameserver /bin/sh -c 'echo nameserver {{ common.master_ip|replace('"','') }} | tee -a /etc/resolvconf/resolv.conf.d/head'
|
||||
|
||||
# configure black module lists
|
||||
# virt-what should be installed
|
||||
if [ ! -f /etc/modprobe.d/blacklist-i2c_piix4.conf ]; then
|
||||
([[ $(virt-what) = "virtualbox" ]] && echo "blacklist i2c_piix4" >> /etc/modprobe.d/blacklist-i2c_piix4.conf || :) && update-initramfs -u -k all
|
||||
modprobe -r i2c_piix4
|
||||
fi
|
||||
|
||||
cloud-init-per instance conntrack_ipv4 /bin/sh -c 'echo nf_conntrack_ipv4 | tee -a /etc/modules'
|
||||
cloud-init-per instance conntrack_ipv6 /bin/sh -c 'echo nf_conntrack_ipv6 | tee -a /etc/modules'
|
||||
cloud-init-per instance conntrack_max /bin/sh -c 'echo "net.nf_conntrack_max=1048576" | tee -a /etc/sysctl.conf'
|
||||
|
||||
cloud-init-per instance conntrack_ipv4_load modprobe nf_conntrack_ipv4
|
||||
cloud-init-per instance conntrack_ipv6_load modprobe nf_conntrack_ipv6
|
||||
cloud-init-per instance conntrack_max_set sysctl -w "net.nf_conntrack_max=1048576"
|
||||
|
||||
cloud-init-per instance dhclient /bin/sh -c 'echo "supersede routers 0;" | tee /etc/dhcp/dhclient.conf'
|
||||
|
||||
# ntp sync
|
||||
# '| tee /dev/null' is needed for returning zero execution code always
|
||||
cloud-init-per instance stop_ntp /bin/sh -c 'service ntp stop | tee /dev/null'
|
||||
cloud-init-per instance sync_date ntpdate -t 4 -b {{ common.master_ip }}
|
||||
cloud-init-per instance sync_hwclock hwclock --systohc
|
||||
|
||||
cloud-init-per instance edit_ntp_conf1 sed -i '/^\s*tinker panic/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf2 sed -i '1 i tinker panic 0' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf_mkdir mkdir -p /var/lib/ntp
|
||||
cloud-init-per instance edit_ntp_conf3 /bin/sh -c 'echo 0 | tee /var/lib/ntp/drift'
|
||||
cloud-init-per instance edit_ntp_conf4 sed -i '/^\s*server/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf5 /bin/sh -c 'echo "server {{ common.master_ip }} burst iburst" | tee -a /etc/ntp.conf'
|
||||
cloud-init-per instance start_ntp service ntp start
|
||||
|
||||
cloud-init-per instance removeUseDNS sed -i --follow-symlinks -e '/UseDNS/d' /etc/ssh/sshd_config
|
||||
add_str_to_file_if_not_exists /etc/ssh/sshd_config 'UseDNS' 'UseDNS no'
|
||||
|
||||
cloud-init-per instance gssapi_disable sed -i -e "/^\s*GSSAPICleanupCredentials yes/d" -e "/^\s*GSSAPIAuthentication yes/d" /etc/ssh/sshd_config
|
||||
|
||||
cloud-init-per instance nailgun_agent_0 /bin/sh -c 'echo "rm -f /etc/nailgun-agent/nodiscover" | tee /etc/rc.local'
|
||||
cloud-init-per instance nailgun_agent_1 /bin/sh -c 'echo "flock -w 0 -o /var/lock/agent.lock -c \"/opt/nailgun/bin/agent >> /var/log/nailgun-agent.log 2>&1\"" | tee -a /etc/rc.local'
|
||||
|
||||
# Copying default bash settings to the root directory
|
||||
cloud-init-per instance skel_bash cp -f /etc/skel/.bash* /root/
|
||||
|
||||
cloud-init-per instance hiera_puppet mkdir -p /etc/puppet /var/lib/hiera
|
||||
cloud-init-per instance touch_puppet touch /var/lib/hiera/common.yaml /etc/puppet/hiera.yaml
|
|
@ -1,104 +0,0 @@
|
|||
#cloud-config
|
||||
resize_rootfs: false
|
||||
growpart:
|
||||
mode: false
|
||||
disable_ec2_metadata: true
|
||||
disable_root: false
|
||||
|
||||
# password: RANDOM
|
||||
# chpasswd: { expire: True }
|
||||
|
||||
ssh_pwauth: false
|
||||
ssh_authorized_keys:
|
||||
{% for key in common.ssh_auth_keys %}
|
||||
- {{ key }}
|
||||
{% endfor %}
|
||||
|
||||
# set the locale to a given locale
|
||||
# default: en_US.UTF-8
|
||||
locale: en_US.UTF-8
|
||||
|
||||
timezone: {{ common.timezone }}
|
||||
|
||||
hostname: {{ common.hostname }}
|
||||
fqdn: {{ common.fqdn }}
|
||||
|
||||
# add entries to rsyslog configuration
|
||||
rsyslog:
|
||||
- filename: 10-log2master.conf
|
||||
content: |
|
||||
$template LogToMaster, "<%%PRI%>1 %$NOW%T%TIMESTAMP:8:$%Z %HOSTNAME% %APP-NAME% %PROCID% %MSGID% -%msg%\n"
|
||||
*.* @{{ common.master_ip }};LogToMaster
|
||||
|
||||
runcmd:
|
||||
{% if puppet.enable != 1 %}
|
||||
- service puppet stop
|
||||
- chkconfig puppet off
|
||||
{% endif %}
|
||||
{% if mcollective.enable != 1 %}
|
||||
- service mcollective stop
|
||||
- chkconfig mcollective off
|
||||
{% else %}
|
||||
- chkconfig mcollective on
|
||||
- service mcollective restart
|
||||
{% endif %}
|
||||
- iptables -t filter -F INPUT
|
||||
- iptables -t filter -F FORWARD
|
||||
- service iptables save
|
||||
|
||||
# that module's missing in 0.6.3, but existent for >= 0.7.3
|
||||
write_files:
|
||||
- content: |
|
||||
---
|
||||
url: {{ common.master_url }}
|
||||
path: /etc/nailgun-agent/config.yaml
|
||||
- content: target
|
||||
path: /etc/nailgun_systemtype
|
||||
|
||||
mcollective:
|
||||
conf:
|
||||
main_collective: mcollective
|
||||
collectives: mcollective
|
||||
libdir: /usr/libexec/mcollective
|
||||
logfile: /var/log/mcollective.log
|
||||
loglevel: debug
|
||||
daemonize: 1
|
||||
direct_addressing: 1
|
||||
ttl: 4294957
|
||||
securityprovider: psk
|
||||
plugin.psk: {{ mcollective.pskey }}
|
||||
{% if mcollective.connector == 'stomp' %}
|
||||
connector = stomp
|
||||
plugin.stomp.host: {{ mcollective.host }}
|
||||
plugin.stomp.port: {{ mcollective.port|default(61613) }}
|
||||
plugin.stomp.user: {{ mcollective.user }}
|
||||
plugin.stomp.password: {{ mcollective.password }}
|
||||
{% else %}
|
||||
connector: rabbitmq
|
||||
plugin.rabbitmq.vhost: {{ mcollective.vhost }}
|
||||
plugin.rabbitmq.pool.size: 1
|
||||
plugin.rabbitmq.pool.1.host: {{ mcollective.host }}
|
||||
plugin.rabbitmq.pool.1.port: {{ mcollective.port|default(61613) }}
|
||||
plugin.rabbitmq.pool.1.user: {{ mcollective.user }}
|
||||
plugin.rabbitmq.pool.1.password: {{ mcollective.password }}
|
||||
plugin.rabbitmq.heartbeat_interval: 30
|
||||
{% endif %}
|
||||
factsource: yaml
|
||||
plugin.yaml: /etc/mcollective/facts.yaml
|
||||
|
||||
puppet:
|
||||
conf:
|
||||
main:
|
||||
logdir: /var/log/puppet
|
||||
rundir: /var/run/puppet
|
||||
ssldir: $vardir/ssl
|
||||
pluginsync: true
|
||||
agent:
|
||||
classfile: $vardir/classes.txt
|
||||
localconfig: $vardir/localconfig
|
||||
server: {{ puppet.master }}
|
||||
report: false
|
||||
configtimeout: 600
|
||||
|
||||
|
||||
final_message: "YAY! The system is finally up, after $UPTIME seconds"
|
|
@ -1,103 +0,0 @@
|
|||
#cloud-config
|
||||
resize_rootfs: false
|
||||
growpart:
|
||||
mode: false
|
||||
disable_ec2_metadata: true
|
||||
disable_root: false
|
||||
user: root
|
||||
password: r00tme
|
||||
chpasswd: { expire: false }
|
||||
ssh_pwauth: false
|
||||
ssh_authorized_keys:
|
||||
{% for key in common.ssh_auth_keys %}
|
||||
- {{ key }}
|
||||
{% endfor %}
|
||||
|
||||
# set the locale to a given locale
|
||||
# default: en_US.UTF-8
|
||||
locale: en_US.UTF-8
|
||||
|
||||
timezone: {{ common.timezone }}
|
||||
|
||||
hostname: {{ common.hostname }}
|
||||
fqdn: {{ common.fqdn }}
|
||||
|
||||
|
||||
# add entries to rsyslog configuration
|
||||
rsyslog:
|
||||
- filename: 10-log2master.conf
|
||||
content: |
|
||||
$template LogToMaster, "<%%PRI%>1 %$NOW%T%TIMESTAMP:8:$%Z %HOSTNAME% %APP-NAME% %PROCID% %MSGID% -%msg%\n"
|
||||
*.* @{{ common.master_ip }};LogToMaster
|
||||
|
||||
|
||||
# that module's missing in 0.6.3, but existent for >= 0.7.3
|
||||
write_files:
|
||||
- content: |
|
||||
---
|
||||
url: {{ common.master_url }}
|
||||
path: /etc/nailgun-agent/config.yaml
|
||||
- content: target
|
||||
path: /etc/nailgun_systemtype
|
||||
|
||||
mcollective:
|
||||
conf:
|
||||
main_collective: mcollective
|
||||
collectives: mcollective
|
||||
libdir: /usr/share/mcollective/plugins
|
||||
logfile: /var/log/mcollective.log
|
||||
loglevel: debug
|
||||
daemonize: 0
|
||||
direct_addressing: 1
|
||||
ttl: 4294957
|
||||
securityprovider: psk
|
||||
plugin.psk: {{ mcollective.pskey }}
|
||||
{% if mcollective.connector == 'stomp' %}
|
||||
connector = stomp
|
||||
plugin.stomp.host: {{ mcollective.host }}
|
||||
plugin.stomp.port: {{ mcollective.port|default(61613) }}
|
||||
plugin.stomp.user: {{ mcollective.user }}
|
||||
plugin.stomp.password: {{ mcollective.password }}
|
||||
{% else %}
|
||||
connector: rabbitmq
|
||||
plugin.rabbitmq.vhost: {{ mcollective.vhost }}
|
||||
plugin.rabbitmq.pool.size: 1
|
||||
plugin.rabbitmq.pool.1.host: {{ mcollective.host }}
|
||||
plugin.rabbitmq.pool.1.port: {{ mcollective.port|default(61613) }}
|
||||
plugin.rabbitmq.pool.1.user: {{ mcollective.user }}
|
||||
plugin.rabbitmq.pool.1.password: {{ mcollective.password }}
|
||||
plugin.rabbitmq.heartbeat_interval: 30
|
||||
{% endif %}
|
||||
factsource: yaml
|
||||
plugin.yaml: /etc/mcollective/facts.yaml
|
||||
|
||||
puppet:
|
||||
conf:
|
||||
main:
|
||||
logdir: /var/log/puppet
|
||||
rundir: /var/run/puppet
|
||||
ssldir: $vardir/ssl
|
||||
pluginsync: true
|
||||
agent:
|
||||
classfile: $vardir/classes.txt
|
||||
localconfig: $vardir/localconfig
|
||||
server: {{ puppet.master }}
|
||||
report: false
|
||||
configtimeout: 600
|
||||
|
||||
runcmd:
|
||||
{% if puppet.enable != 1 %}
|
||||
- /usr/sbin/invoke-rc.d puppet stop
|
||||
- /usr/sbin/update-rc.d -f puppet remove
|
||||
{% endif %}
|
||||
{% if mcollective.enable != 1 %}
|
||||
- /usr/sbin/invoke-rc.d mcollective stop
|
||||
- echo manual > /etc/init/mcollective.override
|
||||
{% else %}
|
||||
- rm -f /etc/init/mcollective.override
|
||||
- service mcollective restart
|
||||
{% endif %}
|
||||
- iptables -t filter -F INPUT
|
||||
- iptables -t filter -F FORWARD
|
||||
|
||||
final_message: "YAY! The system is finally up, after $UPTIME seconds"
|
|
@ -1,11 +0,0 @@
|
|||
# instance-id will be autogenerated
|
||||
# instance-id: iid-abcdefg
|
||||
#network-interfaces: |
|
||||
# auto {{ common.admin_iface_name|default("eth0") }}
|
||||
# iface {{ common.admin_iface_name|default("eth0") }} inet static
|
||||
# address {{ common.admin_ip }}
|
||||
# # network 192.168.1.0
|
||||
# netmask {{ common.admin_mask }}
|
||||
# # broadcast 192.168.1.255
|
||||
# # gateway 192.168.1.254
|
||||
hostname: {{ common.hostname }}
|
|
@ -1,11 +0,0 @@
|
|||
# instance-id will be autogenerated
|
||||
# instance-id: iid-abcdefg
|
||||
#network-interfaces: |
|
||||
# auto {{ common.admin_iface_name|default("eth0") }}
|
||||
# iface {{ common.admin_iface_name|default("eth0") }} inet static
|
||||
# address {{ common.admin_ip }}
|
||||
# # network 192.168.1.0
|
||||
# netmask {{ common.admin_mask }}
|
||||
# # broadcast 192.168.1.255
|
||||
# # gateway 192.168.1.254
|
||||
hostname: {{ common.hostname }}
|
|
@ -1,184 +0,0 @@
|
|||
[DEFAULT]
|
||||
|
||||
#
|
||||
# Options defined in fuel_agent.manager
|
||||
#
|
||||
|
||||
# Data driver (string value)
|
||||
#data_driver=nailgun
|
||||
|
||||
# Path to directory with cloud init templates (string value)
|
||||
#nc_template_path=/usr/share/fuel-agent/cloud-init-templates
|
||||
|
||||
# Temporary directory for file manipulations (string value)
|
||||
#tmp_path=/tmp
|
||||
|
||||
# Path where to store generated config drive image (string
|
||||
# value)
|
||||
#config_drive_path=/tmp/config-drive.img
|
||||
|
||||
# Path where to store actual rules for udev daemon (string
|
||||
# value)
|
||||
#udev_rules_dir=/etc/udev/rules.d
|
||||
|
||||
# Path where to store default rules for udev daemon (string
|
||||
# value)
|
||||
#udev_rules_lib_dir=/lib/udev/rules.d
|
||||
|
||||
# Substring to which file extension .rules be renamed (string
|
||||
# value)
|
||||
#udev_rename_substr=.renamedrule
|
||||
|
||||
# Directory where we build images (string value)
|
||||
#image_build_dir=/tmp
|
||||
|
||||
# Directory where we build images (string value)
|
||||
#image_build_suffix=.fuel-agent-image
|
||||
|
||||
|
||||
#
|
||||
# Options defined in fuel_agent.cmd.agent
|
||||
#
|
||||
|
||||
# Input data file (string value)
|
||||
#input_data_file=/tmp/provision.json
|
||||
|
||||
# Input data (json string) (string value)
|
||||
#input_data=
|
||||
|
||||
|
||||
#
|
||||
# Options defined in fuel_agent.openstack.common.log
|
||||
#
|
||||
|
||||
# Print debugging output (set logging level to DEBUG instead
|
||||
# of default WARNING level). (boolean value)
|
||||
debug=true
|
||||
|
||||
# Print more verbose output (set logging level to INFO instead
|
||||
# of default WARNING level). (boolean value)
|
||||
#verbose=false
|
||||
|
||||
# Log output to standard error. (boolean value)
|
||||
use_stderr=false
|
||||
|
||||
# Format string to use for log messages with context. (string
|
||||
# value)
|
||||
#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
|
||||
|
||||
# Format string to use for log messages without context.
|
||||
# (string value)
|
||||
#logging_default_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s
|
||||
|
||||
# Data to append to log format when level is DEBUG. (string
|
||||
# value)
|
||||
logging_debug_format_suffix=
|
||||
|
||||
# Prefix each line of exception output with this format.
|
||||
# (string value)
|
||||
#logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
|
||||
|
||||
# List of logger=LEVEL pairs. (list value)
|
||||
#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN
|
||||
|
||||
# Enables or disables publication of error events. (boolean
|
||||
# value)
|
||||
#publish_errors=false
|
||||
|
||||
# Enables or disables fatal status of deprecations. (boolean
|
||||
# value)
|
||||
#fatal_deprecations=false
|
||||
|
||||
# The format for an instance that is passed with the log
|
||||
# message. (string value)
|
||||
#instance_format="[instance: %(uuid)s] "
|
||||
|
||||
# The format for an instance UUID that is passed with the log
|
||||
# message. (string value)
|
||||
#instance_uuid_format="[instance: %(uuid)s] "
|
||||
|
||||
# The name of a logging configuration file. This file is
|
||||
# appended to any existing logging configuration files. For
|
||||
# details about logging configuration files, see the Python
|
||||
# logging module documentation. (string value)
|
||||
# Deprecated group/name - [DEFAULT]/log_config
|
||||
#log_config_append=<None>
|
||||
|
||||
# DEPRECATED. A logging.Formatter log message format string
|
||||
# which may use any of the available logging.LogRecord
|
||||
# attributes. This option is deprecated. Please use
|
||||
# logging_context_format_string and
|
||||
# logging_default_format_string instead. (string value)
|
||||
#log_format=<None>
|
||||
|
||||
# Format string for %%(asctime)s in log records. Default:
|
||||
# %(default)s . (string value)
|
||||
#log_date_format=%Y-%m-%d %H:%M:%S
|
||||
|
||||
# (Optional) Name of log file to output to. If no default is
|
||||
# set, logging will go to stdout. (string value)
|
||||
# Deprecated group/name - [DEFAULT]/logfile
|
||||
log_file=/var/log/fuel-agent.log
|
||||
|
||||
|
||||
# (Optional) The base directory used for relative --log-file
|
||||
# paths. (string value)
|
||||
# Deprecated group/name - [DEFAULT]/logdir
|
||||
#log_dir=<None>
|
||||
|
||||
# Use syslog for logging. Existing syslog format is DEPRECATED
|
||||
# during I, and will change in J to honor RFC5424. (boolean
|
||||
# value)
|
||||
#use_syslog=false
|
||||
|
||||
# (Optional) Enables or disables syslog rfc5424 format for
|
||||
# logging. If enabled, prefixes the MSG part of the syslog
|
||||
# message with APP-NAME (RFC5424). The format without the APP-
|
||||
# NAME is deprecated in I, and will be removed in J. (boolean
|
||||
# value)
|
||||
#use_syslog_rfc_format=false
|
||||
|
||||
# Syslog facility to receive log lines. (string value)
|
||||
#syslog_log_facility=LOG_USER
|
||||
|
||||
|
||||
#
|
||||
# Options defined in fuel_agent.utils.artifact
|
||||
#
|
||||
|
||||
# Size of data chunk to operate with images (integer value)
|
||||
#data_chunk_size=1048576
|
||||
|
||||
|
||||
#
|
||||
# Options defined in fuel_agent.utils.build
|
||||
#
|
||||
|
||||
# Maximum allowed loop devices count to use (integer value)
|
||||
#max_loop_count=255
|
||||
|
||||
# Size of sparse file in MiBs (integer value)
|
||||
#sparse_file_size=2048
|
||||
|
||||
# System-wide major number for loop device (integer value)
|
||||
#loop_dev_major=7
|
||||
|
||||
|
||||
#
|
||||
# Options defined in fuel_agent.utils.utils
|
||||
#
|
||||
|
||||
# Maximum retries count for http requests. 0 means infinite
|
||||
# (integer value)
|
||||
#http_max_retries=30
|
||||
|
||||
# Http request timeout in seconds (floating point value)
|
||||
#http_request_timeout=10.0
|
||||
|
||||
# Delay in seconds before the next http request retry
|
||||
# (floating point value)
|
||||
#http_retry_delay=2.0
|
||||
|
||||
# Block size of data to read for calculating checksum (integer
|
||||
# value)
|
||||
#read_chunk_size=1048576
|
|
@ -1,103 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
|
||||
from oslo.config import cfg
|
||||
from oslo_serialization import jsonutils as json
|
||||
import six
|
||||
|
||||
from fuel_agent import manager as manager
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent import version
|
||||
|
||||
cli_opts = [
|
||||
cfg.StrOpt(
|
||||
'input_data_file',
|
||||
default='/tmp/provision.json',
|
||||
help='Input data file'
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'input_data',
|
||||
default='',
|
||||
help='Input data (json string)'
|
||||
),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_cli_opts(cli_opts)
|
||||
|
||||
|
||||
def provision():
|
||||
main(['do_provisioning'])
|
||||
|
||||
|
||||
def partition():
|
||||
main(['do_partitioning'])
|
||||
|
||||
|
||||
def copyimage():
|
||||
main(['do_copyimage'])
|
||||
|
||||
|
||||
def configdrive():
|
||||
main(['do_configdrive'])
|
||||
|
||||
|
||||
def bootloader():
|
||||
main(['do_bootloader'])
|
||||
|
||||
|
||||
def build_image():
|
||||
main(['do_build_image'])
|
||||
|
||||
|
||||
def print_err(line):
|
||||
sys.stderr.write(six.text_type(line))
|
||||
sys.stderr.write('\n')
|
||||
|
||||
|
||||
def handle_exception(exc):
|
||||
LOG = logging.getLogger(__name__)
|
||||
LOG.exception(exc)
|
||||
print_err('Unexpected error')
|
||||
print_err(exc)
|
||||
sys.exit(-1)
|
||||
|
||||
|
||||
def main(actions=None):
|
||||
CONF(sys.argv[1:], project='fuel-agent',
|
||||
version=version.version_info.release_string())
|
||||
|
||||
logging.setup('fuel-agent')
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
if CONF.input_data:
|
||||
data = json.loads(CONF.input_data)
|
||||
else:
|
||||
with open(CONF.input_data_file) as f:
|
||||
data = json.load(f)
|
||||
LOG.debug('Input data: %s', data)
|
||||
|
||||
mgr = manager.Manager(data)
|
||||
if actions:
|
||||
for action in actions:
|
||||
getattr(mgr, action)()
|
||||
except Exception as exc:
|
||||
handle_exception(exc)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
|
@ -1,30 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
import copy
|
||||
|
||||
import six
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class BaseDataDriver(object):
|
||||
"""Data driver API
|
||||
|
||||
For example, data validation methods,
|
||||
methods for getting object schemes, etc.
|
||||
"""
|
||||
|
||||
def __init__(self, data):
|
||||
self.data = copy.deepcopy(data)
|
|
@ -1,149 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import jsonschema
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
KS_SPACES_SCHEMA = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'Partition scheme',
|
||||
'type': 'array',
|
||||
'minItems': 1,
|
||||
'uniqueItems': True,
|
||||
'items': {
|
||||
'anyOf': [
|
||||
{
|
||||
'type': 'object',
|
||||
'required': ['type', 'id', 'volumes', 'name',
|
||||
'size', 'extra', 'free_space'],
|
||||
'properties': {
|
||||
'type': {'enum': ['disk']},
|
||||
'id': {'type': 'string'},
|
||||
'name': {'type': 'string'},
|
||||
'size': {'type': 'integer'},
|
||||
'free_space': {'type': 'integer'},
|
||||
'extra': {
|
||||
'type': 'array',
|
||||
'items': {'type': 'string'},
|
||||
},
|
||||
'volumes': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'anyOf': [
|
||||
{
|
||||
'type': 'object',
|
||||
'required': ['type', 'size',
|
||||
'lvm_meta_size', 'vg'],
|
||||
'properties': {
|
||||
'type': {'enum': ['pv']},
|
||||
'size': {'type': 'integer'},
|
||||
'lvm_meta_size': {'type': 'integer'},
|
||||
'vg': {'type': 'string'}
|
||||
}
|
||||
},
|
||||
{
|
||||
'type': 'object',
|
||||
'required': ['type', 'size'],
|
||||
'properties': {
|
||||
'type': {'enum': ['raid',
|
||||
'partition']},
|
||||
'size': {'type': 'integer'},
|
||||
'mount': {'type': 'string'},
|
||||
'file_system': {'type': 'string'},
|
||||
'name': {'type': 'string'}
|
||||
}
|
||||
},
|
||||
{
|
||||
'type': 'object',
|
||||
'required': ['type', 'size'],
|
||||
'properties': {
|
||||
'type': {'enum': ['boot']},
|
||||
'size': {'type': 'integer'}
|
||||
}
|
||||
},
|
||||
{
|
||||
'type': 'object',
|
||||
'required': ['type', 'size'],
|
||||
'properties': {
|
||||
'type': {'enum': ['lvm_meta_pool']},
|
||||
'size': {'type': 'integer'}
|
||||
}
|
||||
},
|
||||
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
'type': 'object',
|
||||
'required': ['type', 'id', 'volumes'],
|
||||
'properties': {
|
||||
'type': {'enum': ['vg']},
|
||||
'id': {'type': 'string'},
|
||||
'label': {'type': 'string'},
|
||||
'min_size': {'type': 'integer'},
|
||||
'_allocate_size': {'type': 'string'},
|
||||
'volumes': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'object',
|
||||
'required': ['type', 'size', 'name'],
|
||||
'properties': {
|
||||
'type': {'enum': ['lv']},
|
||||
'size': {'type': 'integer'},
|
||||
'name': {'type': 'string'},
|
||||
'mount': {'type': 'string'},
|
||||
'file_system': {'type': 'string'},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def validate(scheme):
|
||||
"""Validates a given partition scheme using jsonschema.
|
||||
|
||||
:param scheme: partition scheme to validate
|
||||
"""
|
||||
try:
|
||||
checker = jsonschema.FormatChecker()
|
||||
jsonschema.validate(scheme, KS_SPACES_SCHEMA,
|
||||
format_checker=checker)
|
||||
except Exception as exc:
|
||||
LOG.exception(exc)
|
||||
raise errors.WrongPartitionSchemeError(str(exc))
|
||||
|
||||
# scheme is not valid if the number of disks is 0
|
||||
if not [d for d in scheme if d['type'] == 'disk']:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Partition scheme seems empty')
|
||||
|
||||
for space in scheme:
|
||||
for volume in space.get('volumes', []):
|
||||
if volume['size'] > 16777216 and volume['mount'] == '/':
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Root file system must be less than 16T')
|
||||
|
||||
# TODO(kozhukalov): need to have additional logical verifications
|
||||
# maybe sizes and format of string values
|
|
@ -1,592 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from six.moves.urllib.parse import urlparse
|
||||
from six.moves.urllib.parse import urlsplit
|
||||
|
||||
from fuel_agent.drivers.base import BaseDataDriver
|
||||
from fuel_agent.drivers import ks_spaces_validator
|
||||
from fuel_agent import errors
|
||||
from fuel_agent import objects
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import hardware as hu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def match_device(hu_disk, ks_disk):
|
||||
"""Check if hu_disk and ks_disk are the same device
|
||||
|
||||
Tries to figure out if hu_disk got from hu.list_block_devices
|
||||
and ks_spaces_disk given correspond to the same disk device. This
|
||||
is the simplified version of hu.match_device
|
||||
|
||||
:param hu_disk: A dict representing disk device how
|
||||
it is given by list_block_devices method.
|
||||
:param ks_disk: A dict representing disk device according to
|
||||
ks_spaces format.
|
||||
|
||||
:returns: True if hu_disk matches ks_spaces_disk else False.
|
||||
"""
|
||||
uspec = hu_disk['uspec']
|
||||
|
||||
# True if at least one by-id link matches ks_disk
|
||||
if ('DEVLINKS' in uspec and len(ks_disk.get('extra', [])) > 0
|
||||
and any(x.startswith('/dev/disk/by-id') for x in
|
||||
set(uspec['DEVLINKS']) &
|
||||
set(['/dev/%s' % l for l in ks_disk['extra']]))):
|
||||
return True
|
||||
|
||||
# True if one of DEVLINKS matches ks_disk id
|
||||
if (len(ks_disk.get('extra', [])) == 0
|
||||
and 'DEVLINKS' in uspec and 'id' in ks_disk
|
||||
and '/dev/%s' % ks_disk['id'] in uspec['DEVLINKS']):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class Nailgun(BaseDataDriver):
|
||||
def __init__(self, data):
|
||||
super(Nailgun, self).__init__(data)
|
||||
|
||||
# this var states whether boot partition
|
||||
# was already allocated on first matching volume
|
||||
# or not
|
||||
self._boot_partition_done = False
|
||||
# this var is used as a flag that /boot fs
|
||||
# has already been added. we need this to
|
||||
# get rid of md over all disks for /boot partition.
|
||||
self._boot_done = False
|
||||
|
||||
self.partition_scheme = self.parse_partition_scheme()
|
||||
self.grub = self.parse_grub()
|
||||
self.configdrive_scheme = self.parse_configdrive_scheme()
|
||||
# parsing image scheme needs partition scheme has been parsed
|
||||
self.image_scheme = self.parse_image_scheme()
|
||||
|
||||
def partition_data(self):
|
||||
return self.data['ks_meta']['pm_data']['ks_spaces']
|
||||
|
||||
@property
|
||||
def ks_disks(self):
|
||||
return filter(
|
||||
lambda x: x['type'] == 'disk' and x['size'] > 0,
|
||||
self.partition_data())
|
||||
|
||||
@property
|
||||
def small_ks_disks(self):
|
||||
"""Get those disks which are smaller than 2T"""
|
||||
return [d for d in self.ks_disks if d['size'] <= 2097152]
|
||||
|
||||
@property
|
||||
def ks_vgs(self):
|
||||
return filter(
|
||||
lambda x: x['type'] == 'vg',
|
||||
self.partition_data())
|
||||
|
||||
@property
|
||||
def hu_disks(self):
|
||||
"""Actual disks which are available on this node
|
||||
|
||||
It is a list of dicts which are formatted other way than
|
||||
ks_spaces disks. To match both of those formats use
|
||||
_match_device method.
|
||||
"""
|
||||
if not getattr(self, '_hu_disks', None):
|
||||
self._hu_disks = hu.list_block_devices(disks=True)
|
||||
return self._hu_disks
|
||||
|
||||
def _disk_dev(self, ks_disk):
|
||||
# first we try to find a device that matches ks_disk
|
||||
# comparing by-id and by-path links
|
||||
matched = [hu_disk['device'] for hu_disk in self.hu_disks
|
||||
if match_device(hu_disk, ks_disk)]
|
||||
# if we can not find a device by its by-id and by-path links
|
||||
# we try to find a device by its name
|
||||
fallback = [hu_disk['device'] for hu_disk in self.hu_disks
|
||||
if '/dev/%s' % ks_disk['name'] == hu_disk['device']]
|
||||
found = matched or fallback
|
||||
if not found or len(found) > 1:
|
||||
raise errors.DiskNotFoundError(
|
||||
'Disk not found: %s' % ks_disk['name'])
|
||||
return found[0]
|
||||
|
||||
def _getlabel(self, label):
|
||||
if not label:
|
||||
return ''
|
||||
# XFS will refuse to format a partition if the
|
||||
# disk label is > 12 characters.
|
||||
return ' -L {0} '.format(label[:12])
|
||||
|
||||
def _get_partition_count(self, name):
|
||||
count = 0
|
||||
for disk in self.ks_disks:
|
||||
count += len([v for v in disk["volumes"]
|
||||
if v.get('name') == name and v['size'] > 0])
|
||||
return count
|
||||
|
||||
def _num_ceph_journals(self):
|
||||
return self._get_partition_count('cephjournal')
|
||||
|
||||
def _num_ceph_osds(self):
|
||||
return self._get_partition_count('ceph')
|
||||
|
||||
def parse_partition_scheme(self):
|
||||
LOG.debug('--- Preparing partition scheme ---')
|
||||
data = self.partition_data()
|
||||
ks_spaces_validator.validate(data)
|
||||
partition_scheme = objects.PartitionScheme()
|
||||
|
||||
ceph_osds = self._num_ceph_osds()
|
||||
journals_left = ceph_osds
|
||||
ceph_journals = self._num_ceph_journals()
|
||||
|
||||
LOG.debug('Looping over all disks in provision data')
|
||||
for disk in self.ks_disks:
|
||||
# skipping disk if there are no volumes with size >0
|
||||
# to be allocated on it which are not boot partitions
|
||||
if all((
|
||||
v["size"] <= 0
|
||||
for v in disk["volumes"]
|
||||
if v["type"] != "boot" and v.get("mount") != "/boot"
|
||||
)):
|
||||
continue
|
||||
LOG.debug('Processing disk %s' % disk['name'])
|
||||
LOG.debug('Adding gpt table on disk %s' % disk['name'])
|
||||
parted = partition_scheme.add_parted(
|
||||
name=self._disk_dev(disk), label='gpt')
|
||||
# we install bootloader on every disk
|
||||
LOG.debug('Adding bootloader stage0 on disk %s' % disk['name'])
|
||||
parted.install_bootloader = True
|
||||
# legacy boot partition
|
||||
LOG.debug('Adding bios_grub partition on disk %s: size=24' %
|
||||
disk['name'])
|
||||
parted.add_partition(size=24, flags=['bios_grub'])
|
||||
# uefi partition (for future use)
|
||||
LOG.debug('Adding UEFI partition on disk %s: size=200' %
|
||||
disk['name'])
|
||||
parted.add_partition(size=200)
|
||||
|
||||
LOG.debug('Looping over all volumes on disk %s' % disk['name'])
|
||||
for volume in disk['volumes']:
|
||||
LOG.debug('Processing volume: '
|
||||
'name=%s type=%s size=%s mount=%s vg=%s' %
|
||||
(volume.get('name'), volume.get('type'),
|
||||
volume.get('size'), volume.get('mount'),
|
||||
volume.get('vg')))
|
||||
if volume['size'] <= 0:
|
||||
LOG.debug('Volume size is zero. Skipping.')
|
||||
continue
|
||||
|
||||
if volume.get('name') == 'cephjournal':
|
||||
LOG.debug('Volume seems to be a CEPH journal volume. '
|
||||
'Special procedure is supposed to be applied.')
|
||||
# We need to allocate a journal partition for each ceph OSD
|
||||
# Determine the number of journal partitions we need on
|
||||
# each device
|
||||
ratio = int(math.ceil(float(ceph_osds) / ceph_journals))
|
||||
|
||||
# No more than 10GB will be allocated to a single journal
|
||||
# partition
|
||||
size = volume["size"] / ratio
|
||||
if size > 10240:
|
||||
size = 10240
|
||||
|
||||
# This will attempt to evenly spread partitions across
|
||||
# multiple devices e.g. 5 osds with 2 journal devices will
|
||||
# create 3 partitions on the first device and 2 on the
|
||||
# second
|
||||
if ratio < journals_left:
|
||||
end = ratio
|
||||
else:
|
||||
end = journals_left
|
||||
|
||||
for i in range(0, end):
|
||||
journals_left -= 1
|
||||
if volume['type'] == 'partition':
|
||||
LOG.debug('Adding CEPH journal partition on '
|
||||
'disk %s: size=%s' %
|
||||
(disk['name'], size))
|
||||
prt = parted.add_partition(size=size)
|
||||
LOG.debug('Partition name: %s' % prt.name)
|
||||
if 'partition_guid' in volume:
|
||||
LOG.debug('Setting partition GUID: %s' %
|
||||
volume['partition_guid'])
|
||||
prt.set_guid(volume['partition_guid'])
|
||||
continue
|
||||
|
||||
if volume['type'] in ('partition', 'pv', 'raid'):
|
||||
if volume.get('mount') != '/boot':
|
||||
LOG.debug('Adding partition on disk %s: size=%s' %
|
||||
(disk['name'], volume['size']))
|
||||
prt = parted.add_partition(size=volume['size'])
|
||||
LOG.debug('Partition name: %s' % prt.name)
|
||||
|
||||
elif volume.get('mount') == '/boot' \
|
||||
and not self._boot_partition_done \
|
||||
and (disk in self.small_ks_disks or
|
||||
not self.small_ks_disks):
|
||||
# NOTE(kozhukalov): On some hardware GRUB is not able
|
||||
# to see disks larger than 2T due to firmware bugs,
|
||||
# so we'd better avoid placing /boot on such
|
||||
# huge disks if it is possible.
|
||||
LOG.debug('Adding /boot partition on disk %s: '
|
||||
'size=%s', disk['name'], volume['size'])
|
||||
prt = parted.add_partition(size=volume['size'])
|
||||
LOG.debug('Partition name: %s', prt.name)
|
||||
self._boot_partition_done = True
|
||||
else:
|
||||
LOG.debug('No need to create partition on disk %s. '
|
||||
'Skipping.', disk['name'])
|
||||
continue
|
||||
|
||||
if volume['type'] == 'partition':
|
||||
if 'partition_guid' in volume:
|
||||
LOG.debug('Setting partition GUID: %s' %
|
||||
volume['partition_guid'])
|
||||
prt.set_guid(volume['partition_guid'])
|
||||
|
||||
if 'mount' in volume and volume['mount'] != 'none':
|
||||
LOG.debug('Adding file system on partition: '
|
||||
'mount=%s type=%s' %
|
||||
(volume['mount'],
|
||||
volume.get('file_system', 'xfs')))
|
||||
partition_scheme.add_fs(
|
||||
device=prt.name, mount=volume['mount'],
|
||||
fs_type=volume.get('file_system', 'xfs'),
|
||||
fs_label=self._getlabel(volume.get('disk_label')))
|
||||
if volume['mount'] == '/boot' and not self._boot_done:
|
||||
self._boot_done = True
|
||||
|
||||
if volume['type'] == 'pv':
|
||||
LOG.debug('Creating pv on partition: pv=%s vg=%s' %
|
||||
(prt.name, volume['vg']))
|
||||
lvm_meta_size = volume.get('lvm_meta_size', 64)
|
||||
# The reason for that is to make sure that
|
||||
# there will be enough space for creating logical volumes.
|
||||
# Default lvm extension size is 4M. Nailgun volume
|
||||
# manager does not care of it and if physical volume size
|
||||
# is 4M * N + 3M and lvm metadata size is 4M * L then only
|
||||
# 4M * (N-L) + 3M of space will be available for
|
||||
# creating logical extensions. So only 4M * (N-L) of space
|
||||
# will be available for logical volumes, while nailgun
|
||||
# volume manager might reguire 4M * (N-L) + 3M
|
||||
# logical volume. Besides, parted aligns partitions
|
||||
# according to its own algorithm and actual partition might
|
||||
# be a bit smaller than integer number of mebibytes.
|
||||
if lvm_meta_size < 10:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Error while creating physical volume: '
|
||||
'lvm metadata size is too small')
|
||||
metadatasize = int(math.floor((lvm_meta_size - 8) / 2))
|
||||
metadatacopies = 2
|
||||
partition_scheme.vg_attach_by_name(
|
||||
pvname=prt.name, vgname=volume['vg'],
|
||||
metadatasize=metadatasize,
|
||||
metadatacopies=metadatacopies)
|
||||
|
||||
if volume['type'] == 'raid':
|
||||
if 'mount' in volume and \
|
||||
volume['mount'] not in ('none', '/boot'):
|
||||
LOG.debug('Attaching partition to RAID '
|
||||
'by its mount point %s' % volume['mount'])
|
||||
partition_scheme.md_attach_by_mount(
|
||||
device=prt.name, mount=volume['mount'],
|
||||
fs_type=volume.get('file_system', 'xfs'),
|
||||
fs_label=self._getlabel(volume.get('disk_label')))
|
||||
|
||||
if 'mount' in volume and volume['mount'] == '/boot' and \
|
||||
not self._boot_done:
|
||||
LOG.debug('Adding file system on partition: '
|
||||
'mount=%s type=%s' %
|
||||
(volume['mount'],
|
||||
volume.get('file_system', 'ext2')))
|
||||
partition_scheme.add_fs(
|
||||
device=prt.name, mount=volume['mount'],
|
||||
fs_type=volume.get('file_system', 'ext2'),
|
||||
fs_label=self._getlabel(volume.get('disk_label')))
|
||||
self._boot_done = True
|
||||
|
||||
# this partition will be used to put there configdrive image
|
||||
if partition_scheme.configdrive_device() is None:
|
||||
LOG.debug('Adding configdrive partition on disk %s: size=20' %
|
||||
disk['name'])
|
||||
parted.add_partition(size=20, configdrive=True)
|
||||
|
||||
# checking if /boot is created
|
||||
if not self._boot_partition_done or not self._boot_done:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'/boot partition has not been created for some reasons')
|
||||
|
||||
LOG.debug('Looping over all volume groups in provision data')
|
||||
for vg in self.ks_vgs:
|
||||
LOG.debug('Processing vg %s' % vg['id'])
|
||||
LOG.debug('Looping over all logical volumes in vg %s' % vg['id'])
|
||||
for volume in vg['volumes']:
|
||||
LOG.debug('Processing lv %s' % volume['name'])
|
||||
if volume['size'] <= 0:
|
||||
LOG.debug('Lv size is zero. Skipping.')
|
||||
continue
|
||||
|
||||
if volume['type'] == 'lv':
|
||||
LOG.debug('Adding lv to vg %s: name=%s, size=%s' %
|
||||
(vg['id'], volume['name'], volume['size']))
|
||||
lv = partition_scheme.add_lv(name=volume['name'],
|
||||
vgname=vg['id'],
|
||||
size=volume['size'])
|
||||
|
||||
if 'mount' in volume and volume['mount'] != 'none':
|
||||
LOG.debug('Adding file system on lv: '
|
||||
'mount=%s type=%s' %
|
||||
(volume['mount'],
|
||||
volume.get('file_system', 'xfs')))
|
||||
partition_scheme.add_fs(
|
||||
device=lv.device_name, mount=volume['mount'],
|
||||
fs_type=volume.get('file_system', 'xfs'),
|
||||
fs_label=self._getlabel(volume.get('disk_label')))
|
||||
|
||||
return partition_scheme
|
||||
|
||||
def parse_configdrive_scheme(self):
|
||||
LOG.debug('--- Preparing configdrive scheme ---')
|
||||
data = self.data
|
||||
configdrive_scheme = objects.ConfigDriveScheme()
|
||||
|
||||
LOG.debug('Adding common parameters')
|
||||
admin_interface = filter(
|
||||
lambda x: (x['mac_address'] ==
|
||||
data['kernel_options']['netcfg/choose_interface']),
|
||||
[dict(name=name, **spec) for name, spec
|
||||
in data['interfaces'].iteritems()])[0]
|
||||
|
||||
ssh_auth_keys = data['ks_meta']['authorized_keys']
|
||||
if data['ks_meta']['auth_key']:
|
||||
ssh_auth_keys.append(data['ks_meta']['auth_key'])
|
||||
|
||||
configdrive_scheme.set_common(
|
||||
ssh_auth_keys=ssh_auth_keys,
|
||||
hostname=data['hostname'],
|
||||
fqdn=data['hostname'],
|
||||
name_servers=data['name_servers'],
|
||||
search_domain=data['name_servers_search'],
|
||||
master_ip=data['ks_meta']['master_ip'],
|
||||
master_url='http://%s:8000/api' % data['ks_meta']['master_ip'],
|
||||
udevrules=data['kernel_options']['udevrules'],
|
||||
admin_mac=data['kernel_options']['netcfg/choose_interface'],
|
||||
admin_ip=admin_interface['ip_address'],
|
||||
admin_mask=admin_interface['netmask'],
|
||||
admin_iface_name=admin_interface['name'],
|
||||
timezone=data['ks_meta'].get('timezone', 'America/Los_Angeles'),
|
||||
gw=data['ks_meta']['gw'],
|
||||
ks_repos=data['ks_meta']['repo_setup']['repos']
|
||||
)
|
||||
|
||||
LOG.debug('Adding puppet parameters')
|
||||
configdrive_scheme.set_puppet(
|
||||
master=data['ks_meta']['puppet_master'],
|
||||
enable=data['ks_meta']['puppet_enable']
|
||||
)
|
||||
|
||||
LOG.debug('Adding mcollective parameters')
|
||||
configdrive_scheme.set_mcollective(
|
||||
pskey=data['ks_meta']['mco_pskey'],
|
||||
vhost=data['ks_meta']['mco_vhost'],
|
||||
host=data['ks_meta']['mco_host'],
|
||||
user=data['ks_meta']['mco_user'],
|
||||
password=data['ks_meta']['mco_password'],
|
||||
connector=data['ks_meta']['mco_connector'],
|
||||
enable=data['ks_meta']['mco_enable']
|
||||
)
|
||||
|
||||
LOG.debug('Setting configdrive profile %s' % data['profile'])
|
||||
configdrive_scheme.set_profile(profile=data['profile'])
|
||||
return configdrive_scheme
|
||||
|
||||
def parse_grub(self):
|
||||
LOG.debug('--- Parse grub settings ---')
|
||||
grub = objects.Grub()
|
||||
LOG.debug('Appending kernel parameters: %s',
|
||||
self.data['ks_meta']['pm_data']['kernel_params'])
|
||||
grub.append_kernel_params(
|
||||
self.data['ks_meta']['pm_data']['kernel_params'])
|
||||
if 'centos' in self.data['profile'].lower() and \
|
||||
not self.data['ks_meta'].get('kernel_lt'):
|
||||
LOG.debug('Prefered kernel version is 2.6')
|
||||
grub.kernel_regexp = r'^vmlinuz-2\.6.*'
|
||||
grub.initrd_regexp = r'^initramfs-2\.6.*'
|
||||
return grub
|
||||
|
||||
def parse_image_scheme(self):
|
||||
LOG.debug('--- Preparing image scheme ---')
|
||||
data = self.data
|
||||
image_scheme = objects.ImageScheme()
|
||||
# FIXME(agordeev): this piece of code for fetching additional image
|
||||
# meta data should be factored out of this particular nailgun driver
|
||||
# into more common and absract data getter which should be able to deal
|
||||
# with various data sources (local file, http(s), etc.) and different
|
||||
# data formats ('blob', json, yaml, etc.).
|
||||
# So, the manager will combine and manipulate all those multiple data
|
||||
# getter instances.
|
||||
# Also, the initial data source should be set to sort out chicken/egg
|
||||
# problem. Command line option may be useful for such a case.
|
||||
# BUG: https://bugs.launchpad.net/fuel/+bug/1430418
|
||||
root_uri = data['ks_meta']['image_data']['/']['uri']
|
||||
filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
|
||||
'.yaml'
|
||||
metadata_url = urljoin(root_uri, filename)
|
||||
try:
|
||||
image_meta = yaml.load(
|
||||
utils.init_http_request(metadata_url).text)
|
||||
except Exception as e:
|
||||
LOG.exception(e)
|
||||
LOG.debug('Failed to fetch/decode image meta data')
|
||||
image_meta = {}
|
||||
# We assume for every file system user may provide a separate
|
||||
# file system image. For example if partitioning scheme has
|
||||
# /, /boot, /var/lib file systems then we will try to get images
|
||||
# for all those mount points. Images data are to be defined
|
||||
# at provision.json -> ['ks_meta']['image_data']
|
||||
LOG.debug('Looping over all images in provision data')
|
||||
for mount_point, image_data in six.iteritems(
|
||||
data['ks_meta']['image_data']):
|
||||
LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
|
||||
(mount_point, image_data['uri'],
|
||||
image_data['format'], image_data['container']))
|
||||
iname = os.path.basename(urlparse(image_data['uri']).path)
|
||||
imeta = next(itertools.chain(
|
||||
(img for img in image_meta.get('images', [])
|
||||
if img['container_name'] == iname), [{}]))
|
||||
image_scheme.add_image(
|
||||
uri=image_data['uri'],
|
||||
target_device=self.partition_scheme.fs_by_mount(
|
||||
mount_point).device,
|
||||
format=image_data['format'],
|
||||
container=image_data['container'],
|
||||
size=imeta.get('raw_size'),
|
||||
md5=imeta.get('raw_md5'),
|
||||
)
|
||||
return image_scheme
|
||||
|
||||
|
||||
class NailgunBuildImage(BaseDataDriver):
|
||||
|
||||
# TODO(kozhukalov):
|
||||
# This list of packages is used by default only if another
|
||||
# list isn't given in build image data. In the future
|
||||
# we need to handle package list in nailgun. Even more,
|
||||
# in the future, we'll be building not only ubuntu images
|
||||
# and we'll likely move this list into some kind of config.
|
||||
DEFAULT_TRUSTY_PACKAGES = [
|
||||
"acl",
|
||||
"anacron",
|
||||
"bash-completion",
|
||||
"bridge-utils",
|
||||
"bsdmainutils",
|
||||
"build-essential",
|
||||
"cloud-init",
|
||||
"curl",
|
||||
"daemonize",
|
||||
"debconf-utils",
|
||||
"gdisk",
|
||||
"grub-pc",
|
||||
"linux-firmware",
|
||||
"linux-firmware-nonfree",
|
||||
"linux-headers-generic-lts-trusty",
|
||||
"linux-image-generic-lts-trusty",
|
||||
"lvm2",
|
||||
"mcollective",
|
||||
"mdadm",
|
||||
"nailgun-agent",
|
||||
"nailgun-mcagents",
|
||||
"nailgun-net-check",
|
||||
"ntp",
|
||||
"openssh-client",
|
||||
"openssh-server",
|
||||
"puppet",
|
||||
"python-amqp",
|
||||
"ruby-augeas",
|
||||
"ruby-ipaddress",
|
||||
"ruby-json",
|
||||
"ruby-netaddr",
|
||||
"ruby-openstack",
|
||||
"ruby-shadow",
|
||||
"ruby-stomp",
|
||||
"telnet",
|
||||
"ubuntu-minimal",
|
||||
"ubuntu-standard",
|
||||
"uuid-runtime",
|
||||
"vim",
|
||||
"virt-what",
|
||||
"vlan",
|
||||
]
|
||||
|
||||
def __init__(self, data):
|
||||
super(NailgunBuildImage, self).__init__(data)
|
||||
self.parse_schemes()
|
||||
self.parse_operating_system()
|
||||
|
||||
def parse_operating_system(self):
|
||||
if self.data.get('codename').lower() != 'trusty':
|
||||
raise errors.WrongInputDataError(
|
||||
'Currently, only Ubuntu Trusty is supported, given '
|
||||
'codename is {0}'.format(self.data.get('codename')))
|
||||
|
||||
packages = self.data.get('packages', self.DEFAULT_TRUSTY_PACKAGES)
|
||||
|
||||
repos = []
|
||||
for repo in self.data['repos']:
|
||||
repos.append(objects.DEBRepo(
|
||||
name=repo['name'],
|
||||
uri=repo['uri'],
|
||||
suite=repo['suite'],
|
||||
section=repo['section'],
|
||||
priority=repo['priority']))
|
||||
|
||||
self.operating_system = objects.Ubuntu(repos=repos, packages=packages)
|
||||
|
||||
def parse_schemes(self):
|
||||
self.image_scheme = objects.ImageScheme()
|
||||
self.partition_scheme = objects.PartitionScheme()
|
||||
|
||||
for mount, image in six.iteritems(self.data['image_data']):
|
||||
filename = os.path.basename(urlsplit(image['uri']).path)
|
||||
# Loop does not allocate any loop device
|
||||
# during initialization.
|
||||
device = objects.Loop()
|
||||
|
||||
self.image_scheme.add_image(
|
||||
uri='file://' + os.path.join(self.data['output'], filename),
|
||||
format=image['format'],
|
||||
container=image['container'],
|
||||
target_device=device)
|
||||
|
||||
self.partition_scheme.add_fs(
|
||||
device=device,
|
||||
mount=mount,
|
||||
fs_type=image['format'])
|
||||
|
||||
if mount == '/':
|
||||
metadata_filename = filename.split('.', 1)[0] + '.yaml'
|
||||
self.metadata_uri = 'file://' + os.path.join(
|
||||
self.data['output'], metadata_filename)
|
|
@ -1,164 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class BaseError(Exception):
|
||||
def __init__(self, message, *args, **kwargs):
|
||||
self.message = message
|
||||
super(BaseError, self).__init__(message, *args, **kwargs)
|
||||
|
||||
|
||||
class WrongInputDataError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class WrongPartitionSchemeError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class WrongPartitionLabelError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class PartitionNotFoundError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class DiskNotFoundError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class NotEnoughSpaceError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class PVAlreadyExistsError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class PVNotFoundError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class PVBelongsToVGError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class VGAlreadyExistsError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class VGNotFoundError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class LVAlreadyExistsError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class LVNotFoundError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class MDAlreadyExistsError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class MDNotFoundError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class MDDeviceDuplicationError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class MDWrongSpecError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class MDRemovingError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class WrongConfigDriveDataError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class WrongImageDataError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class TemplateWriteError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class ProcessExecutionError(BaseError):
|
||||
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
|
||||
description=None):
|
||||
self.exit_code = exit_code
|
||||
self.stderr = stderr
|
||||
self.stdout = stdout
|
||||
self.cmd = cmd
|
||||
self.description = description
|
||||
|
||||
if description is None:
|
||||
description = ("Unexpected error while running command.")
|
||||
if exit_code is None:
|
||||
exit_code = '-'
|
||||
message = ('%(description)s\n'
|
||||
'Command: %(cmd)s\n'
|
||||
'Exit code: %(exit_code)s\n'
|
||||
'Stdout: %(stdout)r\n'
|
||||
'Stderr: %(stderr)r') % {'description': description,
|
||||
'cmd': cmd,
|
||||
'exit_code': exit_code,
|
||||
'stdout': stdout,
|
||||
'stderr': stderr}
|
||||
super(ProcessExecutionError, self).__init__(message)
|
||||
|
||||
|
||||
class GrubUtilsError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class FsUtilsError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class HttpUrlConnectionError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class HttpUrlInvalidContentLength(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class ImageChecksumMismatchError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class NoFreeLoopDevices(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class WrongRepositoryError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class WrongDeviceError(BaseError):
|
||||
pass
|
||||
|
||||
|
||||
class UnexpectedProcessError(BaseError):
|
||||
pass
|
|
@ -1,21 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
def setup_hook(config):
|
||||
import pbr
|
||||
import pbr.packaging
|
||||
|
||||
# this monkey patch is to avoid appending git version to version
|
||||
pbr.packaging._get_version_from_git = lambda pre_version: pre_version
|
|
@ -1,725 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import tempfile
|
||||
import yaml
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import artifact as au
|
||||
from fuel_agent.utils import build as bu
|
||||
from fuel_agent.utils import fs as fu
|
||||
from fuel_agent.utils import grub as gu
|
||||
from fuel_agent.utils import lvm as lu
|
||||
from fuel_agent.utils import md as mu
|
||||
from fuel_agent.utils import partition as pu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
opts = [
|
||||
cfg.StrOpt(
|
||||
'nc_template_path',
|
||||
default='/usr/share/fuel-agent/cloud-init-templates',
|
||||
help='Path to directory with cloud init templates',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'tmp_path',
|
||||
default='/tmp',
|
||||
help='Temporary directory for file manipulations',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'config_drive_path',
|
||||
default='/tmp/config-drive.img',
|
||||
help='Path where to store generated config drive image',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'udev_rules_dir',
|
||||
default='/etc/udev/rules.d',
|
||||
help='Path where to store actual rules for udev daemon',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'udev_rules_lib_dir',
|
||||
default='/lib/udev/rules.d',
|
||||
help='Path where to store default rules for udev daemon',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'udev_rename_substr',
|
||||
default='.renamedrule',
|
||||
help='Substring to which file extension .rules be renamed',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'udev_empty_rule',
|
||||
default='empty_rule',
|
||||
help='Correct empty rule for udev daemon',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'image_build_dir',
|
||||
default='/tmp',
|
||||
help='Directory where the image is supposed to be built',
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'image_build_suffix',
|
||||
default='.fuel-agent-image',
|
||||
help='Suffix which is used while creating temporary files',
|
||||
),
|
||||
]
|
||||
|
||||
cli_opts = [
|
||||
cfg.StrOpt(
|
||||
'data_driver',
|
||||
default='nailgun',
|
||||
help='Data driver'
|
||||
),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(opts)
|
||||
CONF.register_cli_opts(cli_opts)
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, data):
|
||||
self.driver = utils.get_driver(CONF.data_driver)(data)
|
||||
|
||||
def do_partitioning(self):
|
||||
LOG.debug('--- Partitioning disks (do_partitioning) ---')
|
||||
# If disks are not wiped out at all, it is likely they contain lvm
|
||||
# and md metadata which will prevent re-creating a partition table
|
||||
# with 'device is busy' error.
|
||||
mu.mdclean_all()
|
||||
lu.lvremove_all()
|
||||
lu.vgremove_all()
|
||||
lu.pvremove_all()
|
||||
|
||||
# Here is udev's rules blacklisting to be done:
|
||||
# by adding symlinks to /dev/null in /etc/udev/rules.d for already
|
||||
# existent rules in /lib/.
|
||||
# 'parted' generates too many udev events in short period of time
|
||||
# so we should increase processing speed for those events,
|
||||
# otherwise partitioning is doomed.
|
||||
empty_rule_path = os.path.join(CONF.udev_rules_dir,
|
||||
os.path.basename(CONF.udev_empty_rule))
|
||||
with open(empty_rule_path, 'w') as f:
|
||||
f.write('#\n')
|
||||
LOG.debug("Enabling udev's rules blacklisting")
|
||||
for rule in os.listdir(CONF.udev_rules_lib_dir):
|
||||
dst = os.path.join(CONF.udev_rules_dir, rule)
|
||||
if os.path.isdir(dst):
|
||||
continue
|
||||
if dst.endswith('.rules'):
|
||||
# for successful blacklisting already existent file with name
|
||||
# from /etc which overlaps with /lib should be renamed prior
|
||||
# symlink creation.
|
||||
try:
|
||||
if os.path.exists(dst):
|
||||
os.rename(dst, dst[:-len('.rules')] +
|
||||
CONF.udev_rename_substr)
|
||||
except OSError:
|
||||
LOG.debug("Skipping udev rule %s blacklising" % dst)
|
||||
else:
|
||||
os.symlink(empty_rule_path, dst)
|
||||
utils.execute('udevadm', 'control', '--reload-rules',
|
||||
check_exit_code=[0])
|
||||
|
||||
for parted in self.driver.partition_scheme.parteds:
|
||||
for prt in parted.partitions:
|
||||
# We wipe out the beginning of every new partition
|
||||
# right after creating it. It allows us to avoid possible
|
||||
# interactive dialog if some data (metadata or file system)
|
||||
# present on this new partition and it also allows udev not
|
||||
# hanging trying to parse this data.
|
||||
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
|
||||
'seek=%s' % max(prt.begin - 3, 0), 'count=5',
|
||||
'of=%s' % prt.device, check_exit_code=[0])
|
||||
# Also wipe out the ending of every new partition.
|
||||
# Different versions of md stores metadata in different places.
|
||||
# Adding exit code 1 to be accepted as for handling situation
|
||||
# when 'no space left on device' occurs.
|
||||
utils.execute('dd', 'if=/dev/zero', 'bs=1M',
|
||||
'seek=%s' % max(prt.end - 3, 0), 'count=5',
|
||||
'of=%s' % prt.device, check_exit_code=[0, 1])
|
||||
|
||||
for parted in self.driver.partition_scheme.parteds:
|
||||
pu.make_label(parted.name, parted.label)
|
||||
for prt in parted.partitions:
|
||||
pu.make_partition(prt.device, prt.begin, prt.end, prt.type)
|
||||
for flag in prt.flags:
|
||||
pu.set_partition_flag(prt.device, prt.count, flag)
|
||||
if prt.guid:
|
||||
pu.set_gpt_type(prt.device, prt.count, prt.guid)
|
||||
# If any partition to be created doesn't exist it's an error.
|
||||
# Probably it's again 'device or resource busy' issue.
|
||||
if not os.path.exists(prt.name):
|
||||
raise errors.PartitionNotFoundError(
|
||||
'Partition %s not found after creation' % prt.name)
|
||||
|
||||
# disable udev's rules blacklisting
|
||||
LOG.debug("Disabling udev's rules blacklisting")
|
||||
for rule in os.listdir(CONF.udev_rules_dir):
|
||||
src = os.path.join(CONF.udev_rules_dir, rule)
|
||||
if os.path.isdir(src):
|
||||
continue
|
||||
if src.endswith('.rules'):
|
||||
if os.path.islink(src):
|
||||
try:
|
||||
os.remove(src)
|
||||
except OSError:
|
||||
LOG.debug(
|
||||
"Skipping udev rule %s de-blacklisting" % src)
|
||||
elif src.endswith(CONF.udev_rename_substr):
|
||||
try:
|
||||
if os.path.exists(src):
|
||||
os.rename(src, src[:-len(CONF.udev_rename_substr)] +
|
||||
'.rules')
|
||||
except OSError:
|
||||
LOG.debug("Skipping udev rule %s de-blacklisting" % src)
|
||||
utils.execute('udevadm', 'control', '--reload-rules',
|
||||
check_exit_code=[0])
|
||||
# NOTE(agordeev): re-create all the links which were skipped by udev
|
||||
# while blacklisted
|
||||
# NOTE(agordeev): do subsystem match, otherwise it will stuck
|
||||
utils.execute('udevadm', 'trigger', '--subsystem-match=block',
|
||||
check_exit_code=[0])
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
|
||||
# If one creates partitions with the same boundaries as last time,
|
||||
# there might be md and lvm metadata on those partitions. To prevent
|
||||
# failing of creating md and lvm devices we need to make sure
|
||||
# unused metadata are wiped out.
|
||||
mu.mdclean_all()
|
||||
lu.lvremove_all()
|
||||
lu.vgremove_all()
|
||||
lu.pvremove_all()
|
||||
|
||||
# creating meta disks
|
||||
for md in self.driver.partition_scheme.mds:
|
||||
mu.mdcreate(md.name, md.level, *md.devices)
|
||||
|
||||
# creating physical volumes
|
||||
for pv in self.driver.partition_scheme.pvs:
|
||||
lu.pvcreate(pv.name, metadatasize=pv.metadatasize,
|
||||
metadatacopies=pv.metadatacopies)
|
||||
|
||||
# creating volume groups
|
||||
for vg in self.driver.partition_scheme.vgs:
|
||||
lu.vgcreate(vg.name, *vg.pvnames)
|
||||
|
||||
# creating logical volumes
|
||||
for lv in self.driver.partition_scheme.lvs:
|
||||
lu.lvcreate(lv.vgname, lv.name, lv.size)
|
||||
|
||||
# making file systems
|
||||
for fs in self.driver.partition_scheme.fss:
|
||||
found_images = [img for img in self.driver.image_scheme.images
|
||||
if img.target_device == fs.device]
|
||||
if not found_images:
|
||||
fu.make_fs(fs.type, fs.options, fs.label, fs.device)
|
||||
|
||||
def do_configdrive(self):
|
||||
LOG.debug('--- Creating configdrive (do_configdrive) ---')
|
||||
cc_output_path = os.path.join(CONF.tmp_path, 'cloud_config.txt')
|
||||
bh_output_path = os.path.join(CONF.tmp_path, 'boothook.txt')
|
||||
# NOTE:file should be strictly named as 'user-data'
|
||||
# the same is for meta-data as well
|
||||
ud_output_path = os.path.join(CONF.tmp_path, 'user-data')
|
||||
md_output_path = os.path.join(CONF.tmp_path, 'meta-data')
|
||||
|
||||
tmpl_dir = CONF.nc_template_path
|
||||
utils.render_and_save(
|
||||
tmpl_dir,
|
||||
self.driver.configdrive_scheme.template_names('cloud_config'),
|
||||
self.driver.configdrive_scheme.template_data(),
|
||||
cc_output_path
|
||||
)
|
||||
utils.render_and_save(
|
||||
tmpl_dir,
|
||||
self.driver.configdrive_scheme.template_names('boothook'),
|
||||
self.driver.configdrive_scheme.template_data(),
|
||||
bh_output_path
|
||||
)
|
||||
utils.render_and_save(
|
||||
tmpl_dir,
|
||||
self.driver.configdrive_scheme.template_names('meta-data'),
|
||||
self.driver.configdrive_scheme.template_data(),
|
||||
md_output_path
|
||||
)
|
||||
|
||||
utils.execute('write-mime-multipart', '--output=%s' % ud_output_path,
|
||||
'%s:text/cloud-boothook' % bh_output_path,
|
||||
'%s:text/cloud-config' % cc_output_path)
|
||||
utils.execute('genisoimage', '-output', CONF.config_drive_path,
|
||||
'-volid', 'cidata', '-joliet', '-rock', ud_output_path,
|
||||
md_output_path)
|
||||
|
||||
configdrive_device = self.driver.partition_scheme.configdrive_device()
|
||||
if configdrive_device is None:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Error while trying to get configdrive device: '
|
||||
'configdrive device not found')
|
||||
size = os.path.getsize(CONF.config_drive_path)
|
||||
md5 = utils.calculate_md5(CONF.config_drive_path, size)
|
||||
self.driver.image_scheme.add_image(
|
||||
uri='file://%s' % CONF.config_drive_path,
|
||||
target_device=configdrive_device,
|
||||
format='iso9660',
|
||||
container='raw',
|
||||
size=size,
|
||||
md5=md5,
|
||||
)
|
||||
|
||||
def do_copyimage(self):
|
||||
LOG.debug('--- Copying images (do_copyimage) ---')
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Processing image: %s' % image.uri)
|
||||
processing = au.Chain()
|
||||
|
||||
LOG.debug('Appending uri processor: %s' % image.uri)
|
||||
processing.append(image.uri)
|
||||
|
||||
if image.uri.startswith('http://'):
|
||||
LOG.debug('Appending HTTP processor')
|
||||
processing.append(au.HttpUrl)
|
||||
elif image.uri.startswith('file://'):
|
||||
LOG.debug('Appending FILE processor')
|
||||
processing.append(au.LocalFile)
|
||||
|
||||
if image.container == 'gzip':
|
||||
LOG.debug('Appending GZIP processor')
|
||||
processing.append(au.GunzipStream)
|
||||
|
||||
LOG.debug('Appending TARGET processor: %s' % image.target_device)
|
||||
processing.append(image.target_device)
|
||||
|
||||
LOG.debug('Launching image processing chain')
|
||||
processing.process()
|
||||
|
||||
if image.size and image.md5:
|
||||
LOG.debug('Trying to compare image checksum')
|
||||
actual_md5 = utils.calculate_md5(image.target_device,
|
||||
image.size)
|
||||
if actual_md5 == image.md5:
|
||||
LOG.debug('Checksum matches successfully: md5=%s' %
|
||||
actual_md5)
|
||||
else:
|
||||
raise errors.ImageChecksumMismatchError(
|
||||
'Actual checksum %s mismatches with expected %s for '
|
||||
'file %s' % (actual_md5, image.md5,
|
||||
image.target_device))
|
||||
else:
|
||||
LOG.debug('Skipping image checksum comparing. '
|
||||
'Ether size or hash have been missed')
|
||||
|
||||
LOG.debug('Extending image file systems')
|
||||
if image.format in ('ext2', 'ext3', 'ext4', 'xfs'):
|
||||
LOG.debug('Extending %s %s' %
|
||||
(image.format, image.target_device))
|
||||
fu.extend_fs(image.format, image.target_device)
|
||||
|
||||
# TODO(kozhukalov): write tests
|
||||
def mount_target(self, chroot, treat_mtab=True, pseudo=True):
|
||||
"""Mount a set of file systems into a chroot
|
||||
|
||||
:param chroot: Directory where to mount file systems
|
||||
:param treat_mtab: If mtab needs to be actualized (Default: True)
|
||||
:param pseudo: If pseudo file systems
|
||||
need to be mounted (Default: True)
|
||||
"""
|
||||
LOG.debug('Mounting target file systems: %s', chroot)
|
||||
# Here we are going to mount all file systems in partition scheme.
|
||||
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
|
||||
if fs.mount == 'swap':
|
||||
continue
|
||||
mount = chroot + fs.mount
|
||||
utils.makedirs_if_not_exists(mount)
|
||||
fu.mount_fs(fs.type, str(fs.device), mount)
|
||||
|
||||
if pseudo:
|
||||
for path in ('/sys', '/dev', '/proc'):
|
||||
utils.makedirs_if_not_exists(chroot + path)
|
||||
fu.mount_bind(chroot, path)
|
||||
|
||||
if treat_mtab:
|
||||
mtab = utils.execute(
|
||||
'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0]
|
||||
mtab_path = chroot + '/etc/mtab'
|
||||
if os.path.islink(mtab_path):
|
||||
os.remove(mtab_path)
|
||||
with open(mtab_path, 'wb') as f:
|
||||
f.write(mtab)
|
||||
|
||||
# TODO(kozhukalov): write tests for this method
|
||||
def umount_target(self, chroot, pseudo=True, try_lazy_umount=True):
|
||||
LOG.debug('Umounting target file systems: %s', chroot)
|
||||
if pseudo:
|
||||
for path in ('/proc', '/dev', '/sys'):
|
||||
fu.umount_fs(chroot + path, try_lazy_umount=try_lazy_umount)
|
||||
for fs in self.driver.partition_scheme.fs_sorted_by_depth(
|
||||
reverse=True):
|
||||
if fs.mount == 'swap':
|
||||
continue
|
||||
fu.umount_fs(chroot + fs.mount, try_lazy_umount=try_lazy_umount)
|
||||
|
||||
# TODO(kozhukalov): write tests for this method
|
||||
# https://bugs.launchpad.net/fuel/+bug/1449609
|
||||
def do_bootloader(self):
|
||||
LOG.debug('--- Installing bootloader (do_bootloader) ---')
|
||||
chroot = '/tmp/target'
|
||||
self.mount_target(chroot)
|
||||
|
||||
mount2uuid = {}
|
||||
for fs in self.driver.partition_scheme.fss:
|
||||
mount2uuid[fs.mount] = utils.execute(
|
||||
'blkid', '-o', 'value', '-s', 'UUID', fs.device,
|
||||
check_exit_code=[0])[0].strip()
|
||||
|
||||
grub = self.driver.grub
|
||||
|
||||
grub.version = gu.guess_grub_version(chroot=chroot)
|
||||
boot_device = self.driver.partition_scheme.boot_device(grub.version)
|
||||
install_devices = [d.name for d in self.driver.partition_scheme.parteds
|
||||
if d.install_bootloader]
|
||||
|
||||
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
|
||||
|
||||
kernel = grub.kernel_name or \
|
||||
gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
|
||||
initrd = grub.initrd_name or \
|
||||
gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
|
||||
|
||||
if grub.version == 1:
|
||||
gu.grub1_cfg(kernel=kernel, initrd=initrd,
|
||||
kernel_params=grub.kernel_params, chroot=chroot)
|
||||
gu.grub1_install(install_devices, boot_device, chroot=chroot)
|
||||
else:
|
||||
# TODO(kozhukalov): implement which kernel to use by default
|
||||
# Currently only grub1_cfg accepts kernel and initrd parameters.
|
||||
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot)
|
||||
gu.grub2_install(install_devices, chroot=chroot)
|
||||
|
||||
# FIXME(agordeev) There's no convenient way to perfrom NIC remapping in
|
||||
# Ubuntu, so injecting files prior the first boot should work
|
||||
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
|
||||
'w') as f:
|
||||
f.write('# Generated by fuel-agent during provisioning: BEGIN\n')
|
||||
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
|
||||
for mapping in self.driver.configdrive_scheme.\
|
||||
common.udevrules.split(','):
|
||||
mac_addr, nic_name = mapping.split('_')
|
||||
f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
|
||||
'ATTR{address}=="%s", ATTR{type}=="1", KERNEL=="eth*",'
|
||||
' NAME="%s"\n' % (mac_addr, nic_name))
|
||||
f.write('# Generated by fuel-agent during provisioning: END\n')
|
||||
# FIXME(agordeev): Disable net-generator that will add new etries to
|
||||
# 70-persistent-net.rules
|
||||
with open(chroot +
|
||||
'/etc/udev/rules.d/75-persistent-net-generator.rules',
|
||||
'w') as f:
|
||||
f.write('# Generated by fuel-agent during provisioning:\n'
|
||||
'# DO NOT DELETE. It is needed to disable net-generator\n')
|
||||
|
||||
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
|
||||
# This ugly hack is to be used together with the command removing
|
||||
# this lock file not earlier than /etc/rc.local
|
||||
# The reason for this hack to appear is to prevent nailgun-agent from
|
||||
# changing mcollective config at the same time when cloud-init
|
||||
# does the same. Otherwise, we can end up with corrupted mcollective
|
||||
# config. For details see https://bugs.launchpad.net/fuel/+bug/1449186
|
||||
LOG.debug('Preventing nailgun-agent from doing '
|
||||
'anything until it is unlocked')
|
||||
utils.makedirs_if_not_exists(os.path.join(chroot, 'etc/nailgun-agent'))
|
||||
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'), 'w'):
|
||||
pass
|
||||
|
||||
with open(chroot + '/etc/fstab', 'wb') as f:
|
||||
for fs in self.driver.partition_scheme.fss:
|
||||
# TODO(kozhukalov): Think of improving the logic so as to
|
||||
# insert a meaningful fsck order value which is last zero
|
||||
# at fstab line. Currently we set it into 0 which means
|
||||
# a corresponding file system will never be checked. We assume
|
||||
# puppet or other configuration tool will care of it.
|
||||
f.write('UUID=%s %s %s defaults 0 0\n' %
|
||||
(mount2uuid[fs.mount], fs.mount, fs.type))
|
||||
|
||||
self.umount_target(chroot)
|
||||
|
||||
def do_reboot(self):
|
||||
LOG.debug('--- Rebooting node (do_reboot) ---')
|
||||
utils.execute('reboot')
|
||||
|
||||
def do_provisioning(self):
|
||||
LOG.debug('--- Provisioning (do_provisioning) ---')
|
||||
self.do_partitioning()
|
||||
self.do_configdrive()
|
||||
self.do_copyimage()
|
||||
self.do_bootloader()
|
||||
LOG.debug('--- Provisioning END (do_provisioning) ---')
|
||||
|
||||
# TODO(kozhukalov): Split this huge method
|
||||
# into a set of smaller ones
|
||||
# https://bugs.launchpad.net/fuel/+bug/1444090
|
||||
def do_build_image(self):
|
||||
"""Building OS images
|
||||
|
||||
Includes the following steps
|
||||
1) create temporary sparse files for all images (truncate)
|
||||
2) attach temporary files to loop devices (losetup)
|
||||
3) create file systems on these loop devices
|
||||
4) create temporary chroot directory
|
||||
5) mount loop devices into chroot directory
|
||||
6) install operating system (debootstrap and apt-get)
|
||||
7) configure OS (clean sources.list and preferences, etc.)
|
||||
8) umount loop devices
|
||||
9) resize file systems on loop devices
|
||||
10) shrink temporary sparse files (images)
|
||||
11) containerize (gzip) temporary sparse files
|
||||
12) move temporary gzipped files to their final location
|
||||
"""
|
||||
LOG.info('--- Building image (do_build_image) ---')
|
||||
# TODO(kozhukalov): Implement metadata
|
||||
# as a pluggable data driver to avoid any fixed format.
|
||||
metadata = {}
|
||||
|
||||
# TODO(kozhukalov): implement this using image metadata
|
||||
# we need to compare list of packages and repos
|
||||
LOG.info('*** Checking if image exists ***')
|
||||
if all([os.path.exists(img.uri.split('file://', 1)[1])
|
||||
for img in self.driver.image_scheme.images]):
|
||||
LOG.debug('All necessary images are available. '
|
||||
'Nothing needs to be done.')
|
||||
return
|
||||
LOG.debug('At least one of the necessary images is unavailable. '
|
||||
'Starting build process.')
|
||||
try:
|
||||
LOG.debug('Creating temporary chroot directory')
|
||||
chroot = tempfile.mkdtemp(
|
||||
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
|
||||
LOG.debug('Temporary chroot: %s', chroot)
|
||||
|
||||
proc_path = os.path.join(chroot, 'proc')
|
||||
|
||||
LOG.info('*** Preparing image space ***')
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Creating temporary sparsed file for the '
|
||||
'image: %s', image.uri)
|
||||
img_tmp_file = bu.create_sparse_tmp_file(
|
||||
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
|
||||
LOG.debug('Temporary file: %s', img_tmp_file)
|
||||
|
||||
# we need to remember those files
|
||||
# to be able to shrink them and move in the end
|
||||
image.img_tmp_file = img_tmp_file
|
||||
|
||||
LOG.debug('Looking for a free loop device')
|
||||
image.target_device.name = bu.get_free_loop_device()
|
||||
|
||||
LOG.debug('Attaching temporary image file to free loop device')
|
||||
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
|
||||
|
||||
# find fs with the same loop device object
|
||||
# as image.target_device
|
||||
fs = self.driver.partition_scheme.fs_by_device(
|
||||
image.target_device)
|
||||
|
||||
LOG.debug('Creating file system on the image')
|
||||
fu.make_fs(
|
||||
fs_type=fs.type,
|
||||
fs_options=fs.options,
|
||||
fs_label=fs.label,
|
||||
dev=str(fs.device))
|
||||
if fs.type == 'ext4':
|
||||
LOG.debug('Trying to disable journaling for ext4 '
|
||||
'in order to speed up the build')
|
||||
utils.execute('tune2fs', '-O', '^has_journal',
|
||||
str(fs.device))
|
||||
|
||||
# mounting all images into chroot tree
|
||||
self.mount_target(chroot, treat_mtab=False, pseudo=False)
|
||||
|
||||
LOG.info('*** Shipping image content ***')
|
||||
LOG.debug('Installing operating system into image')
|
||||
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
|
||||
|
||||
# DEBOOTSTRAP
|
||||
# we use first repo as the main mirror
|
||||
uri = self.driver.operating_system.repos[0].uri
|
||||
suite = self.driver.operating_system.repos[0].suite
|
||||
|
||||
LOG.debug('Preventing services from being get started')
|
||||
bu.suppress_services_start(chroot)
|
||||
LOG.debug('Installing base operating system using debootstrap')
|
||||
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)
|
||||
|
||||
# APT-GET
|
||||
LOG.debug('Configuring apt inside chroot')
|
||||
LOG.debug('Setting environment variables')
|
||||
bu.set_apt_get_env()
|
||||
LOG.debug('Allowing unauthenticated repos')
|
||||
bu.pre_apt_get(chroot)
|
||||
|
||||
for repo in self.driver.operating_system.repos:
|
||||
LOG.debug('Adding repository source: name={name}, uri={uri},'
|
||||
'suite={suite}, section={section}'.format(
|
||||
name=repo.name, uri=repo.uri,
|
||||
suite=repo.suite, section=repo.section))
|
||||
bu.add_apt_source(
|
||||
name=repo.name,
|
||||
uri=repo.uri,
|
||||
suite=repo.suite,
|
||||
section=repo.section,
|
||||
chroot=chroot)
|
||||
LOG.debug('Adding repository preference: '
|
||||
'name={name}, priority={priority}'.format(
|
||||
name=repo.name, priority=repo.priority))
|
||||
if repo.priority is not None:
|
||||
bu.add_apt_preference(
|
||||
name=repo.name,
|
||||
priority=repo.priority,
|
||||
suite=repo.suite,
|
||||
section=repo.section,
|
||||
chroot=chroot,
|
||||
uri=repo.uri)
|
||||
|
||||
metadata.setdefault('repos', []).append({
|
||||
'type': 'deb',
|
||||
'name': repo.name,
|
||||
'uri': repo.uri,
|
||||
'suite': repo.suite,
|
||||
'section': repo.section,
|
||||
'priority': repo.priority,
|
||||
'meta': repo.meta})
|
||||
|
||||
LOG.debug('Preventing services from being get started')
|
||||
bu.suppress_services_start(chroot)
|
||||
|
||||
packages = self.driver.operating_system.packages
|
||||
metadata['packages'] = packages
|
||||
|
||||
# we need /proc to be mounted for apt-get success
|
||||
utils.makedirs_if_not_exists(proc_path)
|
||||
fu.mount_bind(chroot, '/proc')
|
||||
|
||||
LOG.debug('Installing packages using apt-get: %s',
|
||||
' '.join(packages))
|
||||
bu.run_apt_get(chroot, packages=packages)
|
||||
|
||||
LOG.debug('Post-install OS configuration')
|
||||
bu.do_post_inst(chroot)
|
||||
|
||||
LOG.debug('Making sure there are no running processes '
|
||||
'inside chroot before trying to umount chroot')
|
||||
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
|
||||
if not bu.stop_chrooted_processes(
|
||||
chroot, signal=signal.SIGKILL):
|
||||
raise errors.UnexpectedProcessError(
|
||||
'Stopping chrooted processes failed. '
|
||||
'There are some processes running in chroot %s',
|
||||
chroot)
|
||||
|
||||
LOG.info('*** Finalizing image space ***')
|
||||
fu.umount_fs(proc_path)
|
||||
# umounting all loop devices
|
||||
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
|
||||
|
||||
for image in self.driver.image_scheme.images:
|
||||
# find fs with the same loop device object
|
||||
# as image.target_device
|
||||
fs = self.driver.partition_scheme.fs_by_device(
|
||||
image.target_device)
|
||||
|
||||
if fs.type == 'ext4':
|
||||
LOG.debug('Trying to re-enable journaling for ext4')
|
||||
utils.execute('tune2fs', '-O', 'has_journal',
|
||||
str(fs.device))
|
||||
|
||||
LOG.debug('Deattaching loop device from file: %s',
|
||||
image.img_tmp_file)
|
||||
bu.deattach_loop(str(image.target_device))
|
||||
LOG.debug('Shrinking temporary image file: %s',
|
||||
image.img_tmp_file)
|
||||
bu.shrink_sparse_file(image.img_tmp_file)
|
||||
|
||||
raw_size = os.path.getsize(image.img_tmp_file)
|
||||
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
|
||||
|
||||
LOG.debug('Containerizing temporary image file: %s',
|
||||
image.img_tmp_file)
|
||||
img_tmp_containerized = bu.containerize(
|
||||
image.img_tmp_file, image.container)
|
||||
img_containerized = image.uri.split('file://', 1)[1]
|
||||
|
||||
# NOTE(kozhukalov): implement abstract publisher
|
||||
LOG.debug('Moving image file to the final location: %s',
|
||||
img_containerized)
|
||||
shutil.move(img_tmp_containerized, img_containerized)
|
||||
|
||||
container_size = os.path.getsize(img_containerized)
|
||||
container_md5 = utils.calculate_md5(
|
||||
img_containerized, container_size)
|
||||
metadata.setdefault('images', []).append({
|
||||
'raw_md5': raw_md5,
|
||||
'raw_size': raw_size,
|
||||
'raw_name': None,
|
||||
'container_name': os.path.basename(img_containerized),
|
||||
'container_md5': container_md5,
|
||||
'container_size': container_size,
|
||||
'container': image.container,
|
||||
'format': image.format})
|
||||
|
||||
# NOTE(kozhukalov): implement abstract publisher
|
||||
LOG.debug('Image metadata: %s', metadata)
|
||||
with open(self.driver.metadata_uri.split('file://', 1)[1],
|
||||
'w') as f:
|
||||
yaml.safe_dump(metadata, stream=f)
|
||||
LOG.info('--- Building image END (do_build_image) ---')
|
||||
except Exception as exc:
|
||||
LOG.error('Failed to build image: %s', exc)
|
||||
raise
|
||||
finally:
|
||||
LOG.debug('Finally: stopping processes inside chroot: %s', chroot)
|
||||
|
||||
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
|
||||
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
|
||||
LOG.debug('Finally: umounting procfs %s', proc_path)
|
||||
fu.umount_fs(proc_path)
|
||||
LOG.debug('Finally: umounting chroot tree %s', chroot)
|
||||
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
|
||||
for image in self.driver.image_scheme.images:
|
||||
LOG.debug('Finally: detaching loop device: %s',
|
||||
str(image.target_device))
|
||||
try:
|
||||
bu.deattach_loop(str(image.target_device))
|
||||
except errors.ProcessExecutionError as e:
|
||||
LOG.warning('Error occured while trying to detach '
|
||||
'loop device %s. Error message: %s',
|
||||
str(image.target_device), e)
|
||||
|
||||
LOG.debug('Finally: removing temporary file: %s',
|
||||
image.img_tmp_file)
|
||||
try:
|
||||
os.unlink(image.img_tmp_file)
|
||||
except OSError:
|
||||
LOG.debug('Finally: file %s seems does not exist '
|
||||
'or can not be removed', image.img_tmp_file)
|
||||
LOG.debug('Finally: removing chroot directory: %s', chroot)
|
||||
try:
|
||||
os.rmdir(chroot)
|
||||
except OSError:
|
||||
LOG.debug('Finally: directory %s seems does not exist '
|
||||
'or can not be removed', chroot)
|
|
@ -1,42 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from fuel_agent.objects.bootloader import Grub
|
||||
from fuel_agent.objects.configdrive import ConfigDriveCommon
|
||||
from fuel_agent.objects.configdrive import ConfigDriveMcollective
|
||||
from fuel_agent.objects.configdrive import ConfigDrivePuppet
|
||||
from fuel_agent.objects.configdrive import ConfigDriveScheme
|
||||
from fuel_agent.objects.device import Loop
|
||||
from fuel_agent.objects.image import Image
|
||||
from fuel_agent.objects.image import ImageScheme
|
||||
from fuel_agent.objects.operating_system import OperatingSystem
|
||||
from fuel_agent.objects.operating_system import Ubuntu
|
||||
from fuel_agent.objects.partition import Fs
|
||||
from fuel_agent.objects.partition import Lv
|
||||
from fuel_agent.objects.partition import Md
|
||||
from fuel_agent.objects.partition import Partition
|
||||
from fuel_agent.objects.partition import PartitionScheme
|
||||
from fuel_agent.objects.partition import Pv
|
||||
from fuel_agent.objects.partition import Vg
|
||||
from fuel_agent.objects.repo import DEBRepo
|
||||
from fuel_agent.objects.repo import Repo
|
||||
|
||||
__all__ = [
|
||||
'Partition', 'Pv', 'Vg', 'Lv', 'Md', 'Fs', 'PartitionScheme',
|
||||
'ConfigDriveCommon', 'ConfigDrivePuppet', 'ConfigDriveMcollective',
|
||||
'ConfigDriveScheme', 'Image', 'ImageScheme', 'Grub',
|
||||
'OperatingSystem', 'Ubuntu',
|
||||
'Repo', 'DEBRepo',
|
||||
'Loop',
|
||||
]
|
|
@ -1,29 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class Grub(object):
|
||||
def __init__(self, version=None, kernel_params='',
|
||||
kernel_name=None, kernel_regexp=None,
|
||||
initrd_name=None, initrd_regexp=None):
|
||||
self.version = version
|
||||
self.kernel_params = kernel_params
|
||||
self.kernel_name = kernel_name
|
||||
self.initrd_name = initrd_name
|
||||
self.kernel_regexp = kernel_regexp
|
||||
self.initrd_regexp = initrd_regexp
|
||||
|
||||
def append_kernel_params(self, *kernel_params):
|
||||
for kp in kernel_params:
|
||||
self.kernel_params = '{0} {1}'.format(self.kernel_params, kp)
|
|
@ -1,107 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from fuel_agent import errors
|
||||
|
||||
|
||||
class ConfigDriveCommon(object):
|
||||
def __init__(self, ssh_auth_keys, hostname, fqdn, name_servers,
|
||||
search_domain, master_ip, master_url, udevrules, admin_mac,
|
||||
admin_ip, admin_mask, admin_iface_name, timezone, ks_repos,
|
||||
gw):
|
||||
self.ssh_auth_keys = ssh_auth_keys
|
||||
self.hostname = hostname
|
||||
self.fqdn = fqdn
|
||||
self.name_servers = name_servers
|
||||
self.search_domain = search_domain
|
||||
self.master_ip = master_ip
|
||||
self.master_url = master_url
|
||||
self.udevrules = udevrules
|
||||
self.admin_mac = admin_mac
|
||||
self.admin_ip = admin_ip
|
||||
self.admin_mask = admin_mask
|
||||
self.admin_iface_name = admin_iface_name
|
||||
self.timezone = timezone
|
||||
self.ks_repos = ks_repos
|
||||
self.gw = gw
|
||||
|
||||
|
||||
class ConfigDrivePuppet(object):
|
||||
def __init__(self, master, enable):
|
||||
self.master = master
|
||||
self.enable = enable
|
||||
|
||||
|
||||
class ConfigDriveMcollective(object):
|
||||
def __init__(self, pskey, vhost, host, user, password, connector, enable):
|
||||
self.pskey = pskey
|
||||
self.vhost = vhost
|
||||
self.host = host
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.connector = connector
|
||||
self.enable = enable
|
||||
|
||||
|
||||
class ConfigDriveScheme(object):
|
||||
def __init__(self, common=None, puppet=None,
|
||||
mcollective=None, profile=None):
|
||||
self.common = common
|
||||
self.puppet = puppet
|
||||
self.mcollective = mcollective
|
||||
self._profile = profile or 'ubuntu'
|
||||
|
||||
# TODO(kozhukalov) make it possible to validate scheme according to
|
||||
# chosen profile which means chosen set of cloud-init templates.
|
||||
# In other words make this templating scheme easily extendable.
|
||||
|
||||
def set_common(self, **kwargs):
|
||||
self.common = ConfigDriveCommon(**kwargs)
|
||||
|
||||
def set_puppet(self, **kwargs):
|
||||
self.puppet = ConfigDrivePuppet(**kwargs)
|
||||
|
||||
def set_mcollective(self, **kwargs):
|
||||
self.mcollective = ConfigDriveMcollective(**kwargs)
|
||||
|
||||
def template_data(self):
|
||||
if self.common is None:
|
||||
raise errors.WrongConfigDriveDataError(
|
||||
'Common attribute should be defined, but it is not')
|
||||
template_data = {'common': self.common}
|
||||
if self.puppet is not None:
|
||||
template_data.update(puppet=self.puppet)
|
||||
if self.mcollective is not None:
|
||||
template_data.update(mcollective=self.mcollective)
|
||||
return template_data
|
||||
|
||||
def set_profile(self, profile):
|
||||
# TODO(kozhukalov) validate profile
|
||||
self._profile = profile
|
||||
|
||||
@property
|
||||
def profile(self):
|
||||
return self._profile
|
||||
|
||||
def template_names(self, what):
|
||||
# such a complicated scheme is used to cover a range of profile names
|
||||
# which might be either dash or underline separated
|
||||
# ubuntu_1404_x86_64
|
||||
# centos-65_x86_64
|
||||
return [
|
||||
'%s_%s.jinja2' % (what, self._profile),
|
||||
'%s_%s.jinja2' % (what, self._profile.split('_')[0]),
|
||||
'%s_%s.jinja2' % (what, self._profile.split('-')[0]),
|
||||
'%s.jinja2' % what
|
||||
]
|
|
@ -1,28 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from fuel_agent import errors
|
||||
|
||||
|
||||
class Loop(object):
|
||||
def __init__(self, name=None):
|
||||
self.name = name
|
||||
|
||||
def __str__(self):
|
||||
if self.name:
|
||||
return self.name
|
||||
raise errors.WrongDeviceError(
|
||||
'Loop device can not be stringified. '
|
||||
'Name attribute is not set. Current: '
|
||||
'name={0}'.format(self.name))
|
|
@ -1,44 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from fuel_agent import errors
|
||||
|
||||
|
||||
class Image(object):
|
||||
SUPPORTED_CONTAINERS = ['raw', 'gzip']
|
||||
|
||||
def __init__(self, uri, target_device,
|
||||
format, container, size=None, md5=None):
|
||||
# uri is something like
|
||||
# http://host:port/path/to/image.img or
|
||||
# file:///tmp/image.img
|
||||
self.uri = uri
|
||||
self.target_device = target_device
|
||||
# this must be one of 'iso9660', 'ext[234]', 'xfs'
|
||||
self.format = format
|
||||
if container not in self.SUPPORTED_CONTAINERS:
|
||||
raise errors.WrongImageDataError(
|
||||
'Error while image initialization: '
|
||||
'unsupported image container')
|
||||
self.container = container
|
||||
self.size = size
|
||||
self.md5 = md5
|
||||
|
||||
|
||||
class ImageScheme(object):
|
||||
def __init__(self, images=None):
|
||||
self.images = images or []
|
||||
|
||||
def add_image(self, **kwargs):
|
||||
self.images.append(Image(**kwargs))
|
|
@ -1,23 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class OperatingSystem(object):
|
||||
def __init__(self, repos, packages):
|
||||
self.repos = repos
|
||||
self.packages = packages
|
||||
|
||||
|
||||
class Ubuntu(OperatingSystem):
|
||||
pass
|
|
@ -1,359 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Parted(object):
|
||||
def __init__(self, name, label):
|
||||
self.name = name
|
||||
self.label = label
|
||||
self.partitions = []
|
||||
self.install_bootloader = False
|
||||
|
||||
def add_partition(self, **kwargs):
|
||||
# TODO(kozhukalov): validate before appending
|
||||
# calculating partition name based on device name and partition count
|
||||
kwargs['name'] = self.next_name()
|
||||
kwargs['count'] = self.next_count()
|
||||
kwargs['device'] = self.name
|
||||
# if begin is given use its value else use end of last partition
|
||||
kwargs['begin'] = kwargs.get('begin', self.next_begin())
|
||||
# if end is given use its value else
|
||||
# try to calculate it based on size kwarg or
|
||||
# raise KeyError
|
||||
# (kwargs.pop['size'] will raise error if size is not set)
|
||||
kwargs['end'] = kwargs.get('end') or \
|
||||
kwargs['begin'] + kwargs.pop('size')
|
||||
# if partition_type is given use its value else
|
||||
# try to calculate it automatically
|
||||
kwargs['partition_type'] = \
|
||||
kwargs.get('partition_type', self.next_type())
|
||||
partition = Partition(**kwargs)
|
||||
self.partitions.append(partition)
|
||||
return partition
|
||||
|
||||
@property
|
||||
def logical(self):
|
||||
return filter(lambda x: x.type == 'logical', self.partitions)
|
||||
|
||||
@property
|
||||
def primary(self):
|
||||
return filter(lambda x: x.type == 'primary', self.partitions)
|
||||
|
||||
@property
|
||||
def extended(self):
|
||||
found = filter(lambda x: x.type == 'extended', self.partitions)
|
||||
if found:
|
||||
return found[0]
|
||||
|
||||
def next_type(self):
|
||||
if self.label == 'gpt':
|
||||
return 'primary'
|
||||
elif self.label == 'msdos':
|
||||
if self.extended:
|
||||
return 'logical'
|
||||
elif len(self.partitions) < 3 and not self.extended:
|
||||
return 'primary'
|
||||
elif len(self.partitions) == 3 and not self.extended:
|
||||
return 'extended'
|
||||
# NOTE(agordeev): how to reach that condition?
|
||||
else:
|
||||
return 'logical'
|
||||
|
||||
def next_count(self, next_type=None):
|
||||
next_type = next_type or self.next_type()
|
||||
if next_type == 'logical':
|
||||
return len(self.logical) + 5
|
||||
return len(self.partitions) + 1
|
||||
|
||||
def next_begin(self):
|
||||
if not self.partitions:
|
||||
return 1
|
||||
if self.partitions[-1] == self.extended:
|
||||
return self.partitions[-1].begin
|
||||
return self.partitions[-1].end
|
||||
|
||||
def next_name(self):
|
||||
if self.next_type() == 'extended':
|
||||
return None
|
||||
separator = ''
|
||||
special_devices = ('cciss', 'nvme', 'loop')
|
||||
if any(n in self.name for n in special_devices):
|
||||
separator = 'p'
|
||||
return '%s%s%s' % (self.name, separator, self.next_count())
|
||||
|
||||
|
||||
class Partition(object):
|
||||
def __init__(self, name, count, device, begin, end, partition_type,
|
||||
flags=None, guid=None, configdrive=False):
|
||||
self.name = name
|
||||
self.count = count
|
||||
self.device = device
|
||||
self.name = name
|
||||
self.begin = begin
|
||||
self.end = end
|
||||
self.type = partition_type
|
||||
self.flags = flags or []
|
||||
self.guid = guid
|
||||
self.configdrive = configdrive
|
||||
|
||||
def set_flag(self, flag):
|
||||
if flag not in self.flags:
|
||||
self.flags.append(flag)
|
||||
|
||||
def set_guid(self, guid):
|
||||
self.guid = guid
|
||||
|
||||
|
||||
class Pv(object):
|
||||
def __init__(self, name, metadatasize=16, metadatacopies=2):
|
||||
self.name = name
|
||||
self.metadatasize = metadatasize
|
||||
self.metadatacopies = metadatacopies
|
||||
|
||||
|
||||
class Vg(object):
|
||||
def __init__(self, name, pvnames=None):
|
||||
self.name = name
|
||||
self.pvnames = pvnames or []
|
||||
|
||||
def add_pv(self, pvname):
|
||||
if pvname not in self.pvnames:
|
||||
self.pvnames.append(pvname)
|
||||
|
||||
|
||||
class Lv(object):
|
||||
def __init__(self, name, vgname, size):
|
||||
self.name = name
|
||||
self.vgname = vgname
|
||||
self.size = size
|
||||
|
||||
@property
|
||||
def device_name(self):
|
||||
return '/dev/mapper/%s-%s' % (self.vgname.replace('-', '--'),
|
||||
self.name.replace('-', '--'))
|
||||
|
||||
|
||||
class Md(object):
|
||||
def __init__(self, name, level,
|
||||
devices=None, spares=None):
|
||||
self.name = name
|
||||
self.level = level
|
||||
self.devices = devices or []
|
||||
self.spares = spares or []
|
||||
|
||||
def add_device(self, device):
|
||||
if device in self.devices or device in self.spares:
|
||||
raise errors.MDDeviceDuplicationError(
|
||||
'Error while attaching device to md: '
|
||||
'device %s is already attached' % device)
|
||||
self.devices.append(device)
|
||||
|
||||
def add_spare(self, device):
|
||||
if device in self.devices or device in self.spares:
|
||||
raise errors.MDDeviceDuplicationError(
|
||||
'Error while attaching device to md: '
|
||||
'device %s is already attached' % device)
|
||||
self.spares.append(device)
|
||||
|
||||
|
||||
class Fs(object):
|
||||
def __init__(self, device, mount=None,
|
||||
fs_type=None, fs_options=None, fs_label=None):
|
||||
self.device = device
|
||||
self.mount = mount
|
||||
self.type = fs_type or 'xfs'
|
||||
self.options = fs_options or ''
|
||||
self.label = fs_label or ''
|
||||
|
||||
|
||||
class PartitionScheme(object):
|
||||
def __init__(self):
|
||||
self.parteds = []
|
||||
self.mds = []
|
||||
self.pvs = []
|
||||
self.vgs = []
|
||||
self.lvs = []
|
||||
self.fss = []
|
||||
|
||||
def add_parted(self, **kwargs):
|
||||
parted = Parted(**kwargs)
|
||||
self.parteds.append(parted)
|
||||
return parted
|
||||
|
||||
def add_pv(self, **kwargs):
|
||||
pv = Pv(**kwargs)
|
||||
self.pvs.append(pv)
|
||||
return pv
|
||||
|
||||
def add_vg(self, **kwargs):
|
||||
vg = Vg(**kwargs)
|
||||
self.vgs.append(vg)
|
||||
return vg
|
||||
|
||||
def add_lv(self, **kwargs):
|
||||
lv = Lv(**kwargs)
|
||||
self.lvs.append(lv)
|
||||
return lv
|
||||
|
||||
def add_fs(self, **kwargs):
|
||||
fs = Fs(**kwargs)
|
||||
self.fss.append(fs)
|
||||
return fs
|
||||
|
||||
def add_md(self, **kwargs):
|
||||
mdkwargs = {}
|
||||
mdkwargs['name'] = kwargs.get('name') or self.md_next_name()
|
||||
mdkwargs['level'] = kwargs.get('level') or 'mirror'
|
||||
md = Md(**mdkwargs)
|
||||
self.mds.append(md)
|
||||
return md
|
||||
|
||||
def md_by_name(self, name):
|
||||
found = filter(lambda x: x.name == name, self.mds)
|
||||
if found:
|
||||
return found[0]
|
||||
|
||||
def md_by_mount(self, mount):
|
||||
fs = self.fs_by_mount(mount)
|
||||
if fs:
|
||||
return self.md_by_name(fs.device)
|
||||
|
||||
def md_attach_by_mount(self, device, mount, spare=False, **kwargs):
|
||||
md = self.md_by_mount(mount)
|
||||
if not md:
|
||||
md = self.add_md(**kwargs)
|
||||
fskwargs = {}
|
||||
fskwargs['device'] = md.name
|
||||
fskwargs['mount'] = mount
|
||||
fskwargs['fs_type'] = kwargs.pop('fs_type', None)
|
||||
fskwargs['fs_options'] = kwargs.pop('fs_options', None)
|
||||
fskwargs['fs_label'] = kwargs.pop('fs_label', None)
|
||||
self.add_fs(**fskwargs)
|
||||
md.add_spare(device) if spare else md.add_device(device)
|
||||
return md
|
||||
|
||||
def md_next_name(self):
|
||||
count = 0
|
||||
while True:
|
||||
name = '/dev/md%s' % count
|
||||
if name not in [md.name for md in self.mds]:
|
||||
return name
|
||||
if count >= 127:
|
||||
raise errors.MDAlreadyExistsError(
|
||||
'Error while generating md name: '
|
||||
'names from /dev/md0 to /dev/md127 seem to be busy, '
|
||||
'try to generate md name manually')
|
||||
count += 1
|
||||
|
||||
def vg_by_name(self, vgname):
|
||||
found = filter(lambda x: (x.name == vgname), self.vgs)
|
||||
if found:
|
||||
return found[0]
|
||||
|
||||
def pv_by_name(self, pvname):
|
||||
found = filter(lambda x: (x.name == pvname), self.pvs)
|
||||
if found:
|
||||
return found[0]
|
||||
|
||||
def vg_attach_by_name(self, pvname, vgname,
|
||||
metadatasize=16, metadatacopies=2):
|
||||
vg = self.vg_by_name(vgname) or self.add_vg(name=vgname)
|
||||
pv = self.pv_by_name(pvname) or self.add_pv(
|
||||
name=pvname, metadatasize=metadatasize,
|
||||
metadatacopies=metadatacopies)
|
||||
vg.add_pv(pv.name)
|
||||
|
||||
def fs_by_mount(self, mount):
|
||||
found = filter(lambda x: (x.mount and x.mount == mount), self.fss)
|
||||
if found:
|
||||
return found[0]
|
||||
|
||||
def fs_by_device(self, device):
|
||||
found = filter(lambda x: x.device == device, self.fss)
|
||||
if found:
|
||||
return found[0]
|
||||
|
||||
def fs_sorted_by_depth(self, reverse=False):
|
||||
"""Getting file systems sorted by path length.
|
||||
|
||||
Shorter paths earlier.
|
||||
['/', '/boot', '/var', '/var/lib/mysql']
|
||||
:param reverse: Sort backward (Default: False)
|
||||
"""
|
||||
def key(x):
|
||||
return x.mount.rstrip(os.path.sep).count(os.path.sep)
|
||||
return sorted(self.fss, key=key, reverse=reverse)
|
||||
|
||||
def lv_by_device_name(self, device_name):
|
||||
found = filter(lambda x: x.device_name == device_name, self.lvs)
|
||||
if found:
|
||||
return found[0]
|
||||
|
||||
def root_device(self):
|
||||
fs = self.fs_by_mount('/')
|
||||
if not fs:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Error while trying to find root device: '
|
||||
'root file system not found')
|
||||
return fs.device
|
||||
|
||||
def boot_device(self, grub_version=2):
|
||||
# We assume /boot is a separate partition. If it is not
|
||||
# then we try to use root file system
|
||||
boot_fs = self.fs_by_mount('/boot') or self.fs_by_mount('/')
|
||||
if not boot_fs:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Error while trying to find boot device: '
|
||||
'boot file system not fount, '
|
||||
'it must be a separate mount point')
|
||||
|
||||
if grub_version == 1:
|
||||
# Legacy GRUB has a limitation. It is not able to mount MD devices.
|
||||
# If it is MD compatible it is only able to ignore MD metadata
|
||||
# and to mount one of those devices which are parts of MD device,
|
||||
# but it is possible only if MD device is a MIRROR.
|
||||
md = self.md_by_name(boot_fs.device)
|
||||
if md:
|
||||
try:
|
||||
return md.devices[0]
|
||||
except IndexError:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Error while trying to find boot device: '
|
||||
'md device %s does not have devices attached' %
|
||||
md.name)
|
||||
# Legacy GRUB is not able to mount LVM devices.
|
||||
if self.lv_by_device_name(boot_fs.device):
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Error while trying to find boot device: '
|
||||
'found device is %s but legacy grub is not able to '
|
||||
'mount logical volumes' %
|
||||
boot_fs.device)
|
||||
|
||||
return boot_fs.device
|
||||
|
||||
def configdrive_device(self):
|
||||
# Configdrive device must be a small (about 10M) partition
|
||||
# on one of node hard drives. This partition is necessary
|
||||
# only if one uses cloud-init with configdrive.
|
||||
for parted in self.parteds:
|
||||
for prt in parted.partitions:
|
||||
if prt.configdrive:
|
||||
return prt.name
|
|
@ -1,28 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class Repo(object):
|
||||
def __init__(self, name, uri, priority=None):
|
||||
self.name = name
|
||||
self.uri = uri
|
||||
self.priority = priority
|
||||
|
||||
|
||||
class DEBRepo(Repo):
|
||||
def __init__(self, name, uri, suite, section, meta=None, priority=None):
|
||||
super(DEBRepo, self).__init__(name, uri, priority)
|
||||
self.suite = suite
|
||||
self.section = section
|
||||
self.meta = meta
|
|
@ -1,17 +0,0 @@
|
|||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import six
|
||||
|
||||
|
||||
six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))
|
|
@ -1,314 +0,0 @@
|
|||
# Copyright 2012 SINA Corporation
|
||||
# Copyright 2014 Cisco Systems, Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
"""Extracts OpenStack config option info from module(s)."""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import imp
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
from oslo.config import cfg
|
||||
import six
|
||||
import stevedore.named
|
||||
|
||||
from fuel_agent.openstack.common import gettextutils
|
||||
from fuel_agent.openstack.common import importutils
|
||||
|
||||
gettextutils.install('fuel_agent')
|
||||
|
||||
STROPT = "StrOpt"
|
||||
BOOLOPT = "BoolOpt"
|
||||
INTOPT = "IntOpt"
|
||||
FLOATOPT = "FloatOpt"
|
||||
LISTOPT = "ListOpt"
|
||||
DICTOPT = "DictOpt"
|
||||
MULTISTROPT = "MultiStrOpt"
|
||||
|
||||
OPT_TYPES = {
|
||||
STROPT: 'string value',
|
||||
BOOLOPT: 'boolean value',
|
||||
INTOPT: 'integer value',
|
||||
FLOATOPT: 'floating point value',
|
||||
LISTOPT: 'list value',
|
||||
DICTOPT: 'dict value',
|
||||
MULTISTROPT: 'multi valued',
|
||||
}
|
||||
|
||||
OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT,
|
||||
FLOATOPT, LISTOPT, DICTOPT,
|
||||
MULTISTROPT]))
|
||||
|
||||
PY_EXT = ".py"
|
||||
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||
"../../../../"))
|
||||
WORDWRAP_WIDTH = 60
|
||||
|
||||
|
||||
def raise_extension_exception(extmanager, ep, err):
|
||||
raise
|
||||
|
||||
|
||||
def generate(argv):
|
||||
parser = argparse.ArgumentParser(
|
||||
description='generate sample configuration file',
|
||||
)
|
||||
parser.add_argument('-m', dest='modules', action='append')
|
||||
parser.add_argument('-l', dest='libraries', action='append')
|
||||
parser.add_argument('srcfiles', nargs='*')
|
||||
parsed_args = parser.parse_args(argv)
|
||||
|
||||
mods_by_pkg = dict()
|
||||
for filepath in parsed_args.srcfiles:
|
||||
pkg_name = filepath.split(os.sep)[1]
|
||||
mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]),
|
||||
os.path.basename(filepath).split('.')[0]])
|
||||
mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
|
||||
# NOTE(lzyeval): place top level modules before packages
|
||||
pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT))
|
||||
ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names)
|
||||
pkg_names.extend(ext_names)
|
||||
|
||||
# opts_by_group is a mapping of group name to an options list
|
||||
# The options list is a list of (module, options) tuples
|
||||
opts_by_group = {'DEFAULT': []}
|
||||
|
||||
if parsed_args.modules:
|
||||
for module_name in parsed_args.modules:
|
||||
module = _import_module(module_name)
|
||||
if module:
|
||||
for group, opts in _list_opts(module):
|
||||
opts_by_group.setdefault(group, []).append((module_name,
|
||||
opts))
|
||||
|
||||
# Look for entry points defined in libraries (or applications) for
|
||||
# option discovery, and include their return values in the output.
|
||||
#
|
||||
# Each entry point should be a function returning an iterable
|
||||
# of pairs with the group name (or None for the default group)
|
||||
# and the list of Opt instances for that group.
|
||||
if parsed_args.libraries:
|
||||
loader = stevedore.named.NamedExtensionManager(
|
||||
'oslo.config.opts',
|
||||
names=list(set(parsed_args.libraries)),
|
||||
invoke_on_load=False,
|
||||
on_load_failure_callback=raise_extension_exception
|
||||
)
|
||||
for ext in loader:
|
||||
for group, opts in ext.plugin():
|
||||
opt_list = opts_by_group.setdefault(group or 'DEFAULT', [])
|
||||
opt_list.append((ext.name, opts))
|
||||
|
||||
for pkg_name in pkg_names:
|
||||
mods = mods_by_pkg.get(pkg_name)
|
||||
mods.sort()
|
||||
for mod_str in mods:
|
||||
if mod_str.endswith('.__init__'):
|
||||
mod_str = mod_str[:mod_str.rfind(".")]
|
||||
|
||||
mod_obj = _import_module(mod_str)
|
||||
if not mod_obj:
|
||||
raise RuntimeError("Unable to import module %s" % mod_str)
|
||||
|
||||
for group, opts in _list_opts(mod_obj):
|
||||
opts_by_group.setdefault(group, []).append((mod_str, opts))
|
||||
|
||||
print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', []))
|
||||
for group in sorted(opts_by_group.keys()):
|
||||
print_group_opts(group, opts_by_group[group])
|
||||
|
||||
|
||||
def _import_module(mod_str):
|
||||
try:
|
||||
if mod_str.startswith('bin.'):
|
||||
imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:]))
|
||||
return sys.modules[mod_str[4:]]
|
||||
else:
|
||||
return importutils.import_module(mod_str)
|
||||
except Exception as e:
|
||||
sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e)))
|
||||
return None
|
||||
|
||||
|
||||
def _is_in_group(opt, group):
|
||||
"Check if opt is in group."
|
||||
for value in group._opts.values():
|
||||
# NOTE(llu): Temporary workaround for bug #1262148, wait until
|
||||
# newly released oslo.config support '==' operator.
|
||||
if not(value['opt'] != opt):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _guess_groups(opt, mod_obj):
|
||||
# is it in the DEFAULT group?
|
||||
if _is_in_group(opt, cfg.CONF):
|
||||
return 'DEFAULT'
|
||||
|
||||
# what other groups is it in?
|
||||
for value in cfg.CONF.values():
|
||||
if isinstance(value, cfg.CONF.GroupAttr):
|
||||
if _is_in_group(opt, value._group):
|
||||
return value._group.name
|
||||
|
||||
raise RuntimeError(
|
||||
"Unable to find group for option %s, "
|
||||
"maybe it's defined twice in the same group?"
|
||||
% opt.name
|
||||
)
|
||||
|
||||
|
||||
def _list_opts(obj):
|
||||
def is_opt(o):
|
||||
return (isinstance(o, cfg.Opt) and
|
||||
not isinstance(o, cfg.SubCommandOpt))
|
||||
|
||||
opts = list()
|
||||
for attr_str in dir(obj):
|
||||
attr_obj = getattr(obj, attr_str)
|
||||
if is_opt(attr_obj):
|
||||
opts.append(attr_obj)
|
||||
elif (isinstance(attr_obj, list) and
|
||||
all(map(lambda x: is_opt(x), attr_obj))):
|
||||
opts.extend(attr_obj)
|
||||
|
||||
ret = {}
|
||||
for opt in opts:
|
||||
ret.setdefault(_guess_groups(opt, obj), []).append(opt)
|
||||
return ret.items()
|
||||
|
||||
|
||||
def print_group_opts(group, opts_by_module):
|
||||
print("[%s]" % group)
|
||||
print('')
|
||||
for mod, opts in opts_by_module:
|
||||
print('#')
|
||||
print('# Options defined in %s' % mod)
|
||||
print('#')
|
||||
print('')
|
||||
for opt in opts:
|
||||
_print_opt(opt)
|
||||
print('')
|
||||
|
||||
|
||||
def _get_my_ip():
|
||||
try:
|
||||
csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
csock.connect(('8.8.8.8', 80))
|
||||
(addr, port) = csock.getsockname()
|
||||
csock.close()
|
||||
return addr
|
||||
except socket.error:
|
||||
return None
|
||||
|
||||
|
||||
def _sanitize_default(name, value):
|
||||
"""Set up a reasonably sensible default for pybasedir, my_ip and host."""
|
||||
hostname = socket.gethostname()
|
||||
fqdn = socket.getfqdn()
|
||||
if value.startswith(sys.prefix):
|
||||
# NOTE(jd) Don't use os.path.join, because it is likely to think the
|
||||
# second part is an absolute pathname and therefore drop the first
|
||||
# part.
|
||||
value = os.path.normpath("/usr/" + value[len(sys.prefix):])
|
||||
elif value.startswith(BASEDIR):
|
||||
return value.replace(BASEDIR, '/usr/lib/python/site-packages')
|
||||
elif BASEDIR in value:
|
||||
return value.replace(BASEDIR, '')
|
||||
elif value == _get_my_ip():
|
||||
return '10.0.0.1'
|
||||
elif value in (hostname, fqdn):
|
||||
if 'host' in name:
|
||||
return 'fuel_agent'
|
||||
elif value.endswith(hostname):
|
||||
return value.replace(hostname, 'fuel_agent')
|
||||
elif value.endswith(fqdn):
|
||||
return value.replace(fqdn, 'fuel_agent')
|
||||
elif value.strip() != value:
|
||||
return '"%s"' % value
|
||||
return value
|
||||
|
||||
|
||||
def _print_opt(opt):
|
||||
opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help
|
||||
if not opt_help:
|
||||
sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name)
|
||||
opt_help = ""
|
||||
opt_type = None
|
||||
try:
|
||||
opt_type = OPTION_REGEX.search(str(type(opt))).group(0)
|
||||
except (ValueError, AttributeError) as err:
|
||||
sys.stderr.write("%s\n" % str(err))
|
||||
sys.exit(1)
|
||||
opt_help = u'%s (%s)' % (opt_help,
|
||||
OPT_TYPES[opt_type])
|
||||
print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
|
||||
if opt.deprecated_opts:
|
||||
for deprecated_opt in opt.deprecated_opts:
|
||||
if deprecated_opt.name:
|
||||
deprecated_group = (deprecated_opt.group if
|
||||
deprecated_opt.group else "DEFAULT")
|
||||
print('# Deprecated group/name - [%s]/%s' %
|
||||
(deprecated_group,
|
||||
deprecated_opt.name))
|
||||
try:
|
||||
if opt_default is None:
|
||||
print('#%s=<None>' % opt_name)
|
||||
elif opt_type == STROPT:
|
||||
assert(isinstance(opt_default, six.string_types))
|
||||
print('#%s=%s' % (opt_name, _sanitize_default(opt_name,
|
||||
opt_default)))
|
||||
elif opt_type == BOOLOPT:
|
||||
assert(isinstance(opt_default, bool))
|
||||
print('#%s=%s' % (opt_name, str(opt_default).lower()))
|
||||
elif opt_type == INTOPT:
|
||||
assert(isinstance(opt_default, int) and
|
||||
not isinstance(opt_default, bool))
|
||||
print('#%s=%s' % (opt_name, opt_default))
|
||||
elif opt_type == FLOATOPT:
|
||||
assert(isinstance(opt_default, float))
|
||||
print('#%s=%s' % (opt_name, opt_default))
|
||||
elif opt_type == LISTOPT:
|
||||
assert(isinstance(opt_default, list))
|
||||
print('#%s=%s' % (opt_name, ','.join(opt_default)))
|
||||
elif opt_type == DICTOPT:
|
||||
assert(isinstance(opt_default, dict))
|
||||
opt_default_strlist = [str(key) + ':' + str(value)
|
||||
for (key, value) in opt_default.items()]
|
||||
print('#%s=%s' % (opt_name, ','.join(opt_default_strlist)))
|
||||
elif opt_type == MULTISTROPT:
|
||||
assert(isinstance(opt_default, list))
|
||||
if not opt_default:
|
||||
opt_default = ['']
|
||||
for default in opt_default:
|
||||
print('#%s=%s' % (opt_name, default))
|
||||
print('')
|
||||
except Exception:
|
||||
sys.stderr.write('Error in option "%s"\n' % opt_name)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
generate(sys.argv[1:])
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
|
@ -1,498 +0,0 @@
|
|||
# Copyright 2012 Red Hat, Inc.
|
||||
# Copyright 2013 IBM Corp.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
gettext for openstack-common modules.
|
||||
|
||||
Usual usage in an openstack.common module:
|
||||
|
||||
from fuel_agent.openstack.common.gettextutils import _
|
||||
"""
|
||||
|
||||
import copy
|
||||
import functools
|
||||
import gettext
|
||||
import locale
|
||||
from logging import handlers
|
||||
import os
|
||||
|
||||
from babel import localedata
|
||||
import six
|
||||
|
||||
_AVAILABLE_LANGUAGES = {}
|
||||
|
||||
# FIXME(dhellmann): Remove this when moving to oslo.i18n.
|
||||
USE_LAZY = False
|
||||
|
||||
|
||||
class TranslatorFactory(object):
|
||||
"""Create translator functions
|
||||
"""
|
||||
|
||||
def __init__(self, domain, lazy=False, localedir=None):
|
||||
"""Establish a set of translation functions for the domain.
|
||||
|
||||
:param domain: Name of translation domain,
|
||||
specifying a message catalog.
|
||||
:type domain: str
|
||||
:param lazy: Delays translation until a message is emitted.
|
||||
Defaults to False.
|
||||
:type lazy: Boolean
|
||||
:param localedir: Directory with translation catalogs.
|
||||
:type localedir: str
|
||||
"""
|
||||
self.domain = domain
|
||||
self.lazy = lazy
|
||||
if localedir is None:
|
||||
localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
|
||||
self.localedir = localedir
|
||||
|
||||
def _make_translation_func(self, domain=None):
|
||||
"""Return a new translation function ready for use.
|
||||
|
||||
Takes into account whether or not lazy translation is being
|
||||
done.
|
||||
|
||||
The domain can be specified to override the default from the
|
||||
factory, but the localedir from the factory is always used
|
||||
because we assume the log-level translation catalogs are
|
||||
installed in the same directory as the main application
|
||||
catalog.
|
||||
|
||||
"""
|
||||
if domain is None:
|
||||
domain = self.domain
|
||||
if self.lazy:
|
||||
return functools.partial(Message, domain=domain)
|
||||
t = gettext.translation(
|
||||
domain,
|
||||
localedir=self.localedir,
|
||||
fallback=True,
|
||||
)
|
||||
if six.PY3:
|
||||
return t.gettext
|
||||
return t.ugettext
|
||||
|
||||
@property
|
||||
def primary(self):
|
||||
"The default translation function."
|
||||
return self._make_translation_func()
|
||||
|
||||
def _make_log_translation_func(self, level):
|
||||
return self._make_translation_func(self.domain + '-log-' + level)
|
||||
|
||||
@property
|
||||
def log_info(self):
|
||||
"Translate info-level log messages."
|
||||
return self._make_log_translation_func('info')
|
||||
|
||||
@property
|
||||
def log_warning(self):
|
||||
"Translate warning-level log messages."
|
||||
return self._make_log_translation_func('warning')
|
||||
|
||||
@property
|
||||
def log_error(self):
|
||||
"Translate error-level log messages."
|
||||
return self._make_log_translation_func('error')
|
||||
|
||||
@property
|
||||
def log_critical(self):
|
||||
"Translate critical-level log messages."
|
||||
return self._make_log_translation_func('critical')
|
||||
|
||||
|
||||
# NOTE(dhellmann): When this module moves out of the incubator into
|
||||
# oslo.i18n, these global variables can be moved to an integration
|
||||
# module within each application.
|
||||
|
||||
# Create the global translation functions.
|
||||
_translators = TranslatorFactory('fuel_agent')
|
||||
|
||||
# The primary translation function using the well-known name "_"
|
||||
_ = _translators.primary
|
||||
|
||||
# Translators for log levels.
|
||||
#
|
||||
# The abbreviated names are meant to reflect the usual use of a short
|
||||
# name like '_'. The "L" is for "log" and the other letter comes from
|
||||
# the level.
|
||||
_LI = _translators.log_info
|
||||
_LW = _translators.log_warning
|
||||
_LE = _translators.log_error
|
||||
_LC = _translators.log_critical
|
||||
|
||||
# NOTE(dhellmann): End of globals that will move to the application's
|
||||
# integration module.
|
||||
|
||||
|
||||
def enable_lazy():
|
||||
"""Convenience function for configuring _() to use lazy gettext
|
||||
|
||||
Call this at the start of execution to enable the gettextutils._
|
||||
function to use lazy gettext functionality. This is useful if
|
||||
your project is importing _ directly instead of using the
|
||||
gettextutils.install() way of importing the _ function.
|
||||
"""
|
||||
# FIXME(dhellmann): This function will be removed in oslo.i18n,
|
||||
# because the TranslatorFactory makes it superfluous.
|
||||
global _, _LI, _LW, _LE, _LC, USE_LAZY
|
||||
tf = TranslatorFactory('fuel_agent', lazy=True)
|
||||
_ = tf.primary
|
||||
_LI = tf.log_info
|
||||
_LW = tf.log_warning
|
||||
_LE = tf.log_error
|
||||
_LC = tf.log_critical
|
||||
USE_LAZY = True
|
||||
|
||||
|
||||
def install(domain, lazy=False):
|
||||
"""Install a _() function using the given translation domain.
|
||||
|
||||
Given a translation domain, install a _() function using gettext's
|
||||
install() function.
|
||||
|
||||
The main difference from gettext.install() is that we allow
|
||||
overriding the default localedir (e.g. /usr/share/locale) using
|
||||
a translation-domain-specific environment variable (e.g.
|
||||
NOVA_LOCALEDIR).
|
||||
|
||||
:param domain: the translation domain
|
||||
:param lazy: indicates whether or not to install the lazy _() function.
|
||||
The lazy _() introduces a way to do deferred translation
|
||||
of messages by installing a _ that builds Message objects,
|
||||
instead of strings, which can then be lazily translated into
|
||||
any available locale.
|
||||
"""
|
||||
if lazy:
|
||||
from six import moves
|
||||
tf = TranslatorFactory(domain, lazy=True)
|
||||
moves.builtins.__dict__['_'] = tf.primary
|
||||
else:
|
||||
localedir = '%s_LOCALEDIR' % domain.upper()
|
||||
if six.PY3:
|
||||
gettext.install(domain,
|
||||
localedir=os.environ.get(localedir))
|
||||
else:
|
||||
gettext.install(domain,
|
||||
localedir=os.environ.get(localedir),
|
||||
unicode=True)
|
||||
|
||||
|
||||
class Message(six.text_type):
|
||||
"""A Message object is a unicode object that can be translated.
|
||||
|
||||
Translation of Message is done explicitly using the translate() method.
|
||||
For all non-translation intents and purposes, a Message is simply unicode,
|
||||
and can be treated as such.
|
||||
"""
|
||||
|
||||
def __new__(cls, msgid, msgtext=None, params=None,
|
||||
domain='fuel_agent', *args):
|
||||
"""Create a new Message object.
|
||||
|
||||
In order for translation to work gettext requires a message ID, this
|
||||
msgid will be used as the base unicode text. It is also possible
|
||||
for the msgid and the base unicode text to be different by passing
|
||||
the msgtext parameter.
|
||||
"""
|
||||
# If the base msgtext is not given, we use the default translation
|
||||
# of the msgid (which is in English) just in case the system locale is
|
||||
# not English, so that the base text will be in that locale by default.
|
||||
if not msgtext:
|
||||
msgtext = Message._translate_msgid(msgid, domain)
|
||||
# We want to initialize the parent unicode with the actual object that
|
||||
# would have been plain unicode if 'Message' was not enabled.
|
||||
msg = super(Message, cls).__new__(cls, msgtext)
|
||||
msg.msgid = msgid
|
||||
msg.domain = domain
|
||||
msg.params = params
|
||||
return msg
|
||||
|
||||
def translate(self, desired_locale=None):
|
||||
"""Translate this message to the desired locale.
|
||||
|
||||
:param desired_locale: The desired locale to translate the message to,
|
||||
if no locale is provided the message will be
|
||||
translated to the system's default locale.
|
||||
|
||||
:returns: the translated message in unicode
|
||||
"""
|
||||
|
||||
translated_message = Message._translate_msgid(self.msgid,
|
||||
self.domain,
|
||||
desired_locale)
|
||||
if self.params is None:
|
||||
# No need for more translation
|
||||
return translated_message
|
||||
|
||||
# This Message object may have been formatted with one or more
|
||||
# Message objects as substitution arguments, given either as a single
|
||||
# argument, part of a tuple, or as one or more values in a dictionary.
|
||||
# When translating this Message we need to translate those Messages too
|
||||
translated_params = _translate_args(self.params, desired_locale)
|
||||
|
||||
translated_message = translated_message % translated_params
|
||||
|
||||
return translated_message
|
||||
|
||||
@staticmethod
|
||||
def _translate_msgid(msgid, domain, desired_locale=None):
|
||||
if not desired_locale:
|
||||
system_locale = locale.getdefaultlocale()
|
||||
# If the system locale is not available to the runtime use English
|
||||
if not system_locale[0]:
|
||||
desired_locale = 'en_US'
|
||||
else:
|
||||
desired_locale = system_locale[0]
|
||||
|
||||
locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
|
||||
lang = gettext.translation(domain,
|
||||
localedir=locale_dir,
|
||||
languages=[desired_locale],
|
||||
fallback=True)
|
||||
if six.PY3:
|
||||
translator = lang.gettext
|
||||
else:
|
||||
translator = lang.ugettext
|
||||
|
||||
translated_message = translator(msgid)
|
||||
return translated_message
|
||||
|
||||
def __mod__(self, other):
|
||||
# When we mod a Message we want the actual operation to be performed
|
||||
# by the parent class (i.e. unicode()), the only thing we do here is
|
||||
# save the original msgid and the parameters in case of a translation
|
||||
params = self._sanitize_mod_params(other)
|
||||
unicode_mod = super(Message, self).__mod__(params)
|
||||
modded = Message(self.msgid,
|
||||
msgtext=unicode_mod,
|
||||
params=params,
|
||||
domain=self.domain)
|
||||
return modded
|
||||
|
||||
def _sanitize_mod_params(self, other):
|
||||
"""Sanitize the object being modded with this Message.
|
||||
|
||||
- Add support for modding 'None' so translation supports it
|
||||
- Trim the modded object, which can be a large dictionary, to only
|
||||
those keys that would actually be used in a translation
|
||||
- Snapshot the object being modded, in case the message is
|
||||
translated, it will be used as it was when the Message was created
|
||||
"""
|
||||
if other is None:
|
||||
params = (other,)
|
||||
elif isinstance(other, dict):
|
||||
# Merge the dictionaries
|
||||
# Copy each item in case one does not support deep copy.
|
||||
params = {}
|
||||
if isinstance(self.params, dict):
|
||||
for key, val in self.params.items():
|
||||
params[key] = self._copy_param(val)
|
||||
for key, val in other.items():
|
||||
params[key] = self._copy_param(val)
|
||||
else:
|
||||
params = self._copy_param(other)
|
||||
return params
|
||||
|
||||
def _copy_param(self, param):
|
||||
try:
|
||||
return copy.deepcopy(param)
|
||||
except Exception:
|
||||
# Fallback to casting to unicode this will handle the
|
||||
# python code-like objects that can't be deep-copied
|
||||
return six.text_type(param)
|
||||
|
||||
def __add__(self, other):
|
||||
msg = _('Message objects do not support addition.')
|
||||
raise TypeError(msg)
|
||||
|
||||
def __radd__(self, other):
|
||||
return self.__add__(other)
|
||||
|
||||
if six.PY2:
|
||||
def __str__(self):
|
||||
# NOTE(luisg): Logging in python 2.6 tries to str() log records,
|
||||
# and it expects specifically a UnicodeError in order to proceed.
|
||||
msg = _('Message objects do not support str() because they may '
|
||||
'contain non-ascii characters. '
|
||||
'Please use unicode() or translate() instead.')
|
||||
raise UnicodeError(msg)
|
||||
|
||||
|
||||
def get_available_languages(domain):
|
||||
"""Lists the available languages for the given translation domain.
|
||||
|
||||
:param domain: the domain to get languages for
|
||||
"""
|
||||
if domain in _AVAILABLE_LANGUAGES:
|
||||
return copy.copy(_AVAILABLE_LANGUAGES[domain])
|
||||
|
||||
localedir = '%s_LOCALEDIR' % domain.upper()
|
||||
find = lambda x: gettext.find(domain,
|
||||
localedir=os.environ.get(localedir),
|
||||
languages=[x])
|
||||
|
||||
# NOTE(mrodden): en_US should always be available (and first in case
|
||||
# order matters) since our in-line message strings are en_US
|
||||
language_list = ['en_US']
|
||||
# NOTE(luisg): Babel <1.0 used a function called list(), which was
|
||||
# renamed to locale_identifiers() in >=1.0, the requirements master list
|
||||
# requires >=0.9.6, uncapped, so defensively work with both. We can remove
|
||||
# this check when the master list updates to >=1.0, and update all projects
|
||||
list_identifiers = (getattr(localedata, 'list', None) or
|
||||
getattr(localedata, 'locale_identifiers'))
|
||||
locale_identifiers = list_identifiers()
|
||||
|
||||
for i in locale_identifiers:
|
||||
if find(i) is not None:
|
||||
language_list.append(i)
|
||||
|
||||
# NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported
|
||||
# locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they
|
||||
# are perfectly legitimate locales:
|
||||
# https://github.com/mitsuhiko/babel/issues/37
|
||||
# In Babel 1.3 they fixed the bug and they support these locales, but
|
||||
# they are still not explicitly "listed" by locale_identifiers().
|
||||
# That is why we add the locales here explicitly if necessary so that
|
||||
# they are listed as supported.
|
||||
aliases = {'zh': 'zh_CN',
|
||||
'zh_Hant_HK': 'zh_HK',
|
||||
'zh_Hant': 'zh_TW',
|
||||
'fil': 'tl_PH'}
|
||||
for (locale, alias) in six.iteritems(aliases):
|
||||
if locale in language_list and alias not in language_list:
|
||||
language_list.append(alias)
|
||||
|
||||
_AVAILABLE_LANGUAGES[domain] = language_list
|
||||
return copy.copy(language_list)
|
||||
|
||||
|
||||
def translate(obj, desired_locale=None):
|
||||
"""Gets the translated unicode representation of the given object.
|
||||
|
||||
If the object is not translatable it is returned as-is.
|
||||
If the locale is None the object is translated to the system locale.
|
||||
|
||||
:param obj: the object to translate
|
||||
:param desired_locale: the locale to translate the message to, if None the
|
||||
default system locale will be used
|
||||
:returns: the translated object in unicode, or the original object if
|
||||
it could not be translated
|
||||
"""
|
||||
message = obj
|
||||
if not isinstance(message, Message):
|
||||
# If the object to translate is not already translatable,
|
||||
# let's first get its unicode representation
|
||||
message = six.text_type(obj)
|
||||
if isinstance(message, Message):
|
||||
# Even after unicoding() we still need to check if we are
|
||||
# running with translatable unicode before translating
|
||||
return message.translate(desired_locale)
|
||||
return obj
|
||||
|
||||
|
||||
def _translate_args(args, desired_locale=None):
|
||||
"""Translates all the translatable elements of the given arguments object.
|
||||
|
||||
This method is used for translating the translatable values in method
|
||||
arguments which include values of tuples or dictionaries.
|
||||
If the object is not a tuple or a dictionary the object itself is
|
||||
translated if it is translatable.
|
||||
|
||||
If the locale is None the object is translated to the system locale.
|
||||
|
||||
:param args: the args to translate
|
||||
:param desired_locale: the locale to translate the args to, if None the
|
||||
default system locale will be used
|
||||
:returns: a new args object with the translated contents of the original
|
||||
"""
|
||||
if isinstance(args, tuple):
|
||||
return tuple(translate(v, desired_locale) for v in args)
|
||||
if isinstance(args, dict):
|
||||
translated_dict = {}
|
||||
for (k, v) in six.iteritems(args):
|
||||
translated_v = translate(v, desired_locale)
|
||||
translated_dict[k] = translated_v
|
||||
return translated_dict
|
||||
return translate(args, desired_locale)
|
||||
|
||||
|
||||
class TranslationHandler(handlers.MemoryHandler):
|
||||
"""Handler that translates records before logging them.
|
||||
|
||||
The TranslationHandler takes a locale and a target logging.Handler object
|
||||
to forward LogRecord objects to after translating them. This handler
|
||||
depends on Message objects being logged, instead of regular strings.
|
||||
|
||||
The handler can be configured declaratively in the logging.conf as follows:
|
||||
|
||||
[handlers]
|
||||
keys = translatedlog, translator
|
||||
|
||||
[handler_translatedlog]
|
||||
class = handlers.WatchedFileHandler
|
||||
args = ('/var/log/api-localized.log',)
|
||||
formatter = context
|
||||
|
||||
[handler_translator]
|
||||
class = openstack.common.log.TranslationHandler
|
||||
target = translatedlog
|
||||
args = ('zh_CN',)
|
||||
|
||||
If the specified locale is not available in the system, the handler will
|
||||
log in the default locale.
|
||||
"""
|
||||
|
||||
def __init__(self, locale=None, target=None):
|
||||
"""Initialize a TranslationHandler
|
||||
|
||||
:param locale: locale to use for translating messages
|
||||
:param target: logging.Handler object to forward
|
||||
LogRecord objects to after translation
|
||||
"""
|
||||
# NOTE(luisg): In order to allow this handler to be a wrapper for
|
||||
# other handlers, such as a FileHandler, and still be able to
|
||||
# configure it using logging.conf, this handler has to extend
|
||||
# MemoryHandler because only the MemoryHandlers' logging.conf
|
||||
# parsing is implemented such that it accepts a target handler.
|
||||
handlers.MemoryHandler.__init__(self, capacity=0, target=target)
|
||||
self.locale = locale
|
||||
|
||||
def setFormatter(self, fmt):
|
||||
self.target.setFormatter(fmt)
|
||||
|
||||
def emit(self, record):
|
||||
# We save the message from the original record to restore it
|
||||
# after translation, so other handlers are not affected by this
|
||||
original_msg = record.msg
|
||||
original_args = record.args
|
||||
|
||||
try:
|
||||
self._translate_and_log_record(record)
|
||||
finally:
|
||||
record.msg = original_msg
|
||||
record.args = original_args
|
||||
|
||||
def _translate_and_log_record(self, record):
|
||||
record.msg = translate(record.msg, self.locale)
|
||||
|
||||
# In addition to translating the message, we also need to translate
|
||||
# arguments that were passed to the log method that were not part
|
||||
# of the main message e.g., log.info(_('Some message %s'), this_one))
|
||||
record.args = _translate_args(record.args, self.locale)
|
||||
|
||||
self.target.emit(record)
|
|
@ -1,73 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Import related utilities and helper functions.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
|
||||
def import_class(import_str):
|
||||
"""Returns a class from a string including module and class."""
|
||||
mod_str, _sep, class_str = import_str.rpartition('.')
|
||||
__import__(mod_str)
|
||||
try:
|
||||
return getattr(sys.modules[mod_str], class_str)
|
||||
except AttributeError:
|
||||
raise ImportError('Class %s cannot be found (%s)' %
|
||||
(class_str,
|
||||
traceback.format_exception(*sys.exc_info())))
|
||||
|
||||
|
||||
def import_object(import_str, *args, **kwargs):
|
||||
"""Import a class and return an instance of it."""
|
||||
return import_class(import_str)(*args, **kwargs)
|
||||
|
||||
|
||||
def import_object_ns(name_space, import_str, *args, **kwargs):
|
||||
"""Tries to import object from default namespace.
|
||||
|
||||
Imports a class and return an instance of it, first by trying
|
||||
to find the class in a default namespace, then failing back to
|
||||
a full path if not found in the default namespace.
|
||||
"""
|
||||
import_value = "%s.%s" % (name_space, import_str)
|
||||
try:
|
||||
return import_class(import_value)(*args, **kwargs)
|
||||
except ImportError:
|
||||
return import_class(import_str)(*args, **kwargs)
|
||||
|
||||
|
||||
def import_module(import_str):
|
||||
"""Import a module."""
|
||||
__import__(import_str)
|
||||
return sys.modules[import_str]
|
||||
|
||||
|
||||
def import_versioned_module(version, submodule=None):
|
||||
module = 'fuel_agent.v%s' % version
|
||||
if submodule:
|
||||
module = '.'.join((module, submodule))
|
||||
return import_module(module)
|
||||
|
||||
|
||||
def try_import(import_str, default=None):
|
||||
"""Try to import a module and if it fails return default."""
|
||||
try:
|
||||
return import_module(import_str)
|
||||
except ImportError:
|
||||
return default
|
|
@ -1,45 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Local storage of variables using weak references"""
|
||||
|
||||
import threading
|
||||
import weakref
|
||||
|
||||
|
||||
class WeakLocal(threading.local):
|
||||
def __getattribute__(self, attr):
|
||||
rval = super(WeakLocal, self).__getattribute__(attr)
|
||||
if rval:
|
||||
# NOTE(mikal): this bit is confusing. What is stored is a weak
|
||||
# reference, not the value itself. We therefore need to lookup
|
||||
# the weak reference and return the inner value here.
|
||||
rval = rval()
|
||||
return rval
|
||||
|
||||
def __setattr__(self, attr, value):
|
||||
value = weakref.ref(value)
|
||||
return super(WeakLocal, self).__setattr__(attr, value)
|
||||
|
||||
|
||||
# NOTE(mikal): the name "store" should be deprecated in the future
|
||||
store = WeakLocal()
|
||||
|
||||
# A "weak" store uses weak references and allows an object to fall out of scope
|
||||
# when it falls out of scope in the code that uses the thread local storage. A
|
||||
# "strong" store will hold a reference to the object so that it never falls out
|
||||
# of scope.
|
||||
weak_store = WeakLocal()
|
||||
strong_store = threading.local()
|
|
@ -1,723 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""OpenStack logging handler.
|
||||
|
||||
This module adds to logging functionality by adding the option to specify
|
||||
a context object when calling the various log methods. If the context object
|
||||
is not specified, default formatting is used. Additionally, an instance uuid
|
||||
may be passed as part of the log message, which is intended to make it easier
|
||||
for admins to find messages related to a specific instance.
|
||||
|
||||
It also allows setting of formatting information through conf.
|
||||
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import itertools
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from oslo.config import cfg
|
||||
from oslo_serialization import jsonutils
|
||||
import six
|
||||
from six import moves
|
||||
|
||||
from fuel_agent.openstack.common.gettextutils import _
|
||||
from fuel_agent.openstack.common import importutils
|
||||
from fuel_agent.openstack.common import local
|
||||
|
||||
|
||||
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
|
||||
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
|
||||
|
||||
# NOTE(ldbragst): Let's build a list of regex objects using the list of
|
||||
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
|
||||
# to the list of _SANITIZE_KEYS and we can generate regular expressions
|
||||
# for XML and JSON automatically.
|
||||
_SANITIZE_PATTERNS = []
|
||||
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
|
||||
r'(<%(key)s>).*?(</%(key)s>)',
|
||||
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
|
||||
'.*?([\'"])',
|
||||
r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
|
||||
|
||||
for key in _SANITIZE_KEYS:
|
||||
for pattern in _FORMAT_PATTERNS:
|
||||
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
|
||||
_SANITIZE_PATTERNS.append(reg_ex)
|
||||
|
||||
|
||||
common_cli_opts = [
|
||||
cfg.BoolOpt('debug',
|
||||
short='d',
|
||||
default=False,
|
||||
help='Print debugging output (set logging level to '
|
||||
'DEBUG instead of default WARNING level).'),
|
||||
cfg.BoolOpt('verbose',
|
||||
short='v',
|
||||
default=False,
|
||||
help='Print more verbose output (set logging level to '
|
||||
'INFO instead of default WARNING level).'),
|
||||
]
|
||||
|
||||
logging_cli_opts = [
|
||||
cfg.StrOpt('log-config-append',
|
||||
metavar='PATH',
|
||||
deprecated_name='log-config',
|
||||
help='The name of a logging configuration file. This file '
|
||||
'is appended to any existing logging configuration '
|
||||
'files. For details about logging configuration files, '
|
||||
'see the Python logging module documentation.'),
|
||||
cfg.StrOpt('log-format',
|
||||
metavar='FORMAT',
|
||||
help='DEPRECATED. '
|
||||
'A logging.Formatter log message format string which may '
|
||||
'use any of the available logging.LogRecord attributes. '
|
||||
'This option is deprecated. Please use '
|
||||
'logging_context_format_string and '
|
||||
'logging_default_format_string instead.'),
|
||||
cfg.StrOpt('log-date-format',
|
||||
default=_DEFAULT_LOG_DATE_FORMAT,
|
||||
metavar='DATE_FORMAT',
|
||||
help='Format string for %%(asctime)s in log records. '
|
||||
'Default: %(default)s .'),
|
||||
cfg.StrOpt('log-file',
|
||||
metavar='PATH',
|
||||
deprecated_name='logfile',
|
||||
help='(Optional) Name of log file to output to. '
|
||||
'If no default is set, logging will go to stdout.'),
|
||||
cfg.StrOpt('log-dir',
|
||||
deprecated_name='logdir',
|
||||
help='(Optional) The base directory used for relative '
|
||||
'--log-file paths.'),
|
||||
cfg.BoolOpt('use-syslog',
|
||||
default=False,
|
||||
help='Use syslog for logging. '
|
||||
'Existing syslog format is DEPRECATED during I, '
|
||||
'and will change in J to honor RFC5424.'),
|
||||
cfg.BoolOpt('use-syslog-rfc-format',
|
||||
# TODO(bogdando) remove or use True after existing
|
||||
# syslog format deprecation in J
|
||||
default=False,
|
||||
help='(Optional) Enables or disables syslog rfc5424 format '
|
||||
'for logging. If enabled, prefixes the MSG part of the '
|
||||
'syslog message with APP-NAME (RFC5424). The '
|
||||
'format without the APP-NAME is deprecated in I, '
|
||||
'and will be removed in J.'),
|
||||
cfg.StrOpt('syslog-log-facility',
|
||||
default='LOG_USER',
|
||||
help='Syslog facility to receive log lines.')
|
||||
]
|
||||
|
||||
generic_log_opts = [
|
||||
cfg.BoolOpt('use_stderr',
|
||||
default=True,
|
||||
help='Log output to standard error.')
|
||||
]
|
||||
|
||||
log_opts = [
|
||||
cfg.StrOpt('logging_context_format_string',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||
'%(name)s [%(request_id)s %(user_identity)s] '
|
||||
'%(instance)s%(message)s',
|
||||
help='Format string to use for log messages with context.'),
|
||||
cfg.StrOpt('logging_default_format_string',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||
'%(name)s [-] %(instance)s%(message)s',
|
||||
help='Format string to use for log messages without context.'),
|
||||
cfg.StrOpt('logging_debug_format_suffix',
|
||||
default='%(funcName)s %(pathname)s:%(lineno)d',
|
||||
help='Data to append to log format when level is DEBUG.'),
|
||||
cfg.StrOpt('logging_exception_prefix',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
|
||||
'%(instance)s',
|
||||
help='Prefix each line of exception output with this format.'),
|
||||
cfg.ListOpt('default_log_levels',
|
||||
default=[
|
||||
'amqp=WARN',
|
||||
'amqplib=WARN',
|
||||
'boto=WARN',
|
||||
'qpid=WARN',
|
||||
'sqlalchemy=WARN',
|
||||
'suds=INFO',
|
||||
'oslo.messaging=INFO',
|
||||
'iso8601=WARN',
|
||||
'requests.packages.urllib3.connectionpool=WARN'
|
||||
],
|
||||
help='List of logger=LEVEL pairs.'),
|
||||
cfg.BoolOpt('publish_errors',
|
||||
default=False,
|
||||
help='Enables or disables publication of error events.'),
|
||||
cfg.BoolOpt('fatal_deprecations',
|
||||
default=False,
|
||||
help='Enables or disables fatal status of deprecations.'),
|
||||
|
||||
# NOTE(mikal): there are two options here because sometimes we are handed
|
||||
# a full instance (and could include more information), and other times we
|
||||
# are just handed a UUID for the instance.
|
||||
cfg.StrOpt('instance_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='The format for an instance that is passed with the log '
|
||||
'message. '),
|
||||
cfg.StrOpt('instance_uuid_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='The format for an instance UUID that is passed with the '
|
||||
'log message. '),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_cli_opts(common_cli_opts)
|
||||
CONF.register_cli_opts(logging_cli_opts)
|
||||
CONF.register_opts(generic_log_opts)
|
||||
CONF.register_opts(log_opts)
|
||||
|
||||
# our new audit level
|
||||
# NOTE(jkoelker) Since we synthesized an audit level, make the logging
|
||||
# module aware of it so it acts like other levels.
|
||||
logging.AUDIT = logging.INFO + 1
|
||||
logging.addLevelName(logging.AUDIT, 'AUDIT')
|
||||
|
||||
|
||||
try:
|
||||
NullHandler = logging.NullHandler
|
||||
except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7
|
||||
class NullHandler(logging.Handler):
|
||||
def handle(self, record):
|
||||
pass
|
||||
|
||||
def emit(self, record):
|
||||
pass
|
||||
|
||||
def createLock(self):
|
||||
self.lock = None
|
||||
|
||||
|
||||
def _dictify_context(context):
|
||||
if context is None:
|
||||
return None
|
||||
if not isinstance(context, dict) and getattr(context, 'to_dict', None):
|
||||
context = context.to_dict()
|
||||
return context
|
||||
|
||||
|
||||
def _get_binary_name():
|
||||
return os.path.basename(inspect.stack()[-1][1])
|
||||
|
||||
|
||||
def _get_log_file_path(binary=None):
|
||||
logfile = CONF.log_file
|
||||
logdir = CONF.log_dir
|
||||
|
||||
if logfile and not logdir:
|
||||
return logfile
|
||||
|
||||
if logfile and logdir:
|
||||
return os.path.join(logdir, logfile)
|
||||
|
||||
if logdir:
|
||||
binary = binary or _get_binary_name()
|
||||
return '%s.log' % (os.path.join(logdir, binary),)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def mask_password(message, secret="***"):
|
||||
"""Replace password with 'secret' in message.
|
||||
|
||||
:param message: The string which includes security information.
|
||||
:param secret: value with which to replace passwords.
|
||||
:returns: The unicode value of message with the password fields masked.
|
||||
|
||||
For example:
|
||||
|
||||
>>> mask_password("'adminPass' : 'aaaaa'")
|
||||
"'adminPass' : '***'"
|
||||
>>> mask_password("'admin_pass' : 'aaaaa'")
|
||||
"'admin_pass' : '***'"
|
||||
>>> mask_password('"password" : "aaaaa"')
|
||||
'"password" : "***"'
|
||||
>>> mask_password("'original_password' : 'aaaaa'")
|
||||
"'original_password' : '***'"
|
||||
>>> mask_password("u'original_password' : u'aaaaa'")
|
||||
"u'original_password' : u'***'"
|
||||
"""
|
||||
message = six.text_type(message)
|
||||
|
||||
# NOTE(ldbragst): Check to see if anything in message contains any key
|
||||
# specified in _SANITIZE_KEYS, if not then just return the message since
|
||||
# we don't have to mask any passwords.
|
||||
if not any(key in message for key in _SANITIZE_KEYS):
|
||||
return message
|
||||
|
||||
secret = r'\g<1>' + secret + r'\g<2>'
|
||||
for pattern in _SANITIZE_PATTERNS:
|
||||
message = re.sub(pattern, secret, message)
|
||||
return message
|
||||
|
||||
|
||||
class BaseLoggerAdapter(logging.LoggerAdapter):
|
||||
|
||||
def audit(self, msg, *args, **kwargs):
|
||||
self.log(logging.AUDIT, msg, *args, **kwargs)
|
||||
|
||||
|
||||
class LazyAdapter(BaseLoggerAdapter):
|
||||
def __init__(self, name='unknown', version='unknown'):
|
||||
self._logger = None
|
||||
self.extra = {}
|
||||
self.name = name
|
||||
self.version = version
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
if not self._logger:
|
||||
self._logger = getLogger(self.name, self.version)
|
||||
return self._logger
|
||||
|
||||
|
||||
class ContextAdapter(BaseLoggerAdapter):
|
||||
warn = logging.LoggerAdapter.warning
|
||||
|
||||
def __init__(self, logger, project_name, version_string):
|
||||
self.logger = logger
|
||||
self.project = project_name
|
||||
self.version = version_string
|
||||
self._deprecated_messages_sent = dict()
|
||||
|
||||
@property
|
||||
def handlers(self):
|
||||
return self.logger.handlers
|
||||
|
||||
def deprecated(self, msg, *args, **kwargs):
|
||||
"""Call this method when a deprecated feature is used.
|
||||
|
||||
If the system is configured for fatal deprecations then the message
|
||||
is logged at the 'critical' level and :class:`DeprecatedConfig` will
|
||||
be raised.
|
||||
|
||||
Otherwise, the message will be logged (once) at the 'warn' level.
|
||||
|
||||
:raises: :class:`DeprecatedConfig` if the system is configured for
|
||||
fatal deprecations.
|
||||
|
||||
"""
|
||||
stdmsg = _("Deprecated: %s") % msg
|
||||
if CONF.fatal_deprecations:
|
||||
self.critical(stdmsg, *args, **kwargs)
|
||||
raise DeprecatedConfig(msg=stdmsg)
|
||||
|
||||
# Using a list because a tuple with dict can't be stored in a set.
|
||||
sent_args = self._deprecated_messages_sent.setdefault(msg, list())
|
||||
|
||||
if args in sent_args:
|
||||
# Already logged this message, so don't log it again.
|
||||
return
|
||||
|
||||
sent_args.append(args)
|
||||
self.warn(stdmsg, *args, **kwargs)
|
||||
|
||||
def process(self, msg, kwargs):
|
||||
# NOTE(mrodden): catch any Message/other object and
|
||||
# coerce to unicode before they can get
|
||||
# to the python logging and possibly
|
||||
# cause string encoding trouble
|
||||
if not isinstance(msg, six.string_types):
|
||||
msg = six.text_type(msg)
|
||||
|
||||
if 'extra' not in kwargs:
|
||||
kwargs['extra'] = {}
|
||||
extra = kwargs['extra']
|
||||
|
||||
context = kwargs.pop('context', None)
|
||||
if not context:
|
||||
context = getattr(local.store, 'context', None)
|
||||
if context:
|
||||
extra.update(_dictify_context(context))
|
||||
|
||||
instance = kwargs.pop('instance', None)
|
||||
instance_uuid = (extra.get('instance_uuid') or
|
||||
kwargs.pop('instance_uuid', None))
|
||||
instance_extra = ''
|
||||
if instance:
|
||||
instance_extra = CONF.instance_format % instance
|
||||
elif instance_uuid:
|
||||
instance_extra = (CONF.instance_uuid_format
|
||||
% {'uuid': instance_uuid})
|
||||
extra['instance'] = instance_extra
|
||||
|
||||
extra.setdefault('user_identity', kwargs.pop('user_identity', None))
|
||||
|
||||
extra['project'] = self.project
|
||||
extra['version'] = self.version
|
||||
extra['extra'] = extra.copy()
|
||||
return msg, kwargs
|
||||
|
||||
|
||||
class JSONFormatter(logging.Formatter):
|
||||
def __init__(self, fmt=None, datefmt=None):
|
||||
# NOTE(jkoelker) we ignore the fmt argument, but its still there
|
||||
# since logging.config.fileConfig passes it.
|
||||
self.datefmt = datefmt
|
||||
|
||||
def formatException(self, ei, strip_newlines=True):
|
||||
lines = traceback.format_exception(*ei)
|
||||
if strip_newlines:
|
||||
lines = [moves.filter(
|
||||
lambda x: x,
|
||||
line.rstrip().splitlines()) for line in lines]
|
||||
lines = list(itertools.chain(*lines))
|
||||
return lines
|
||||
|
||||
def format(self, record):
|
||||
message = {'message': record.getMessage(),
|
||||
'asctime': self.formatTime(record, self.datefmt),
|
||||
'name': record.name,
|
||||
'msg': record.msg,
|
||||
'args': record.args,
|
||||
'levelname': record.levelname,
|
||||
'levelno': record.levelno,
|
||||
'pathname': record.pathname,
|
||||
'filename': record.filename,
|
||||
'module': record.module,
|
||||
'lineno': record.lineno,
|
||||
'funcname': record.funcName,
|
||||
'created': record.created,
|
||||
'msecs': record.msecs,
|
||||
'relative_created': record.relativeCreated,
|
||||
'thread': record.thread,
|
||||
'thread_name': record.threadName,
|
||||
'process_name': record.processName,
|
||||
'process': record.process,
|
||||
'traceback': None}
|
||||
|
||||
if hasattr(record, 'extra'):
|
||||
message['extra'] = record.extra
|
||||
|
||||
if record.exc_info:
|
||||
message['traceback'] = self.formatException(record.exc_info)
|
||||
|
||||
return jsonutils.dumps(message)
|
||||
|
||||
|
||||
def _create_logging_excepthook(product_name):
|
||||
def logging_excepthook(exc_type, value, tb):
|
||||
extra = {'exc_info': (exc_type, value, tb)}
|
||||
getLogger(product_name).critical(
|
||||
"".join(traceback.format_exception_only(exc_type, value)),
|
||||
**extra)
|
||||
return logging_excepthook
|
||||
|
||||
|
||||
class LogConfigError(Exception):
|
||||
|
||||
message = _('Error loading logging config %(log_config)s: %(err_msg)s')
|
||||
|
||||
def __init__(self, log_config, err_msg):
|
||||
self.log_config = log_config
|
||||
self.err_msg = err_msg
|
||||
|
||||
def __str__(self):
|
||||
return self.message % dict(log_config=self.log_config,
|
||||
err_msg=self.err_msg)
|
||||
|
||||
|
||||
def _load_log_config(log_config_append):
|
||||
try:
|
||||
logging.config.fileConfig(log_config_append,
|
||||
disable_existing_loggers=False)
|
||||
except moves.configparser.Error as exc:
|
||||
raise LogConfigError(log_config_append, six.text_type(exc))
|
||||
|
||||
|
||||
def setup(product_name, version='unknown'):
|
||||
"""Setup logging."""
|
||||
if CONF.log_config_append:
|
||||
_load_log_config(CONF.log_config_append)
|
||||
else:
|
||||
_setup_logging_from_conf(product_name, version)
|
||||
sys.excepthook = _create_logging_excepthook(product_name)
|
||||
|
||||
|
||||
def set_defaults(logging_context_format_string):
|
||||
cfg.set_defaults(log_opts,
|
||||
logging_context_format_string=
|
||||
logging_context_format_string)
|
||||
|
||||
|
||||
def _find_facility_from_conf():
|
||||
facility_names = logging.handlers.SysLogHandler.facility_names
|
||||
facility = getattr(logging.handlers.SysLogHandler,
|
||||
CONF.syslog_log_facility,
|
||||
None)
|
||||
|
||||
if facility is None and CONF.syslog_log_facility in facility_names:
|
||||
facility = facility_names.get(CONF.syslog_log_facility)
|
||||
|
||||
if facility is None:
|
||||
valid_facilities = facility_names.keys()
|
||||
consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
|
||||
'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
|
||||
'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
|
||||
'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
|
||||
'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
|
||||
valid_facilities.extend(consts)
|
||||
raise TypeError(_('syslog facility must be one of: %s') %
|
||||
', '.join("'%s'" % fac
|
||||
for fac in valid_facilities))
|
||||
|
||||
return facility
|
||||
|
||||
|
||||
class RFCSysLogHandler(logging.handlers.SysLogHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.binary_name = _get_binary_name()
|
||||
# Do not use super() unless type(logging.handlers.SysLogHandler)
|
||||
# is 'type' (Python 2.7).
|
||||
# Use old style calls, if the type is 'classobj' (Python 2.6)
|
||||
logging.handlers.SysLogHandler.__init__(self, *args, **kwargs)
|
||||
|
||||
def format(self, record):
|
||||
# Do not use super() unless type(logging.handlers.SysLogHandler)
|
||||
# is 'type' (Python 2.7).
|
||||
# Use old style calls, if the type is 'classobj' (Python 2.6)
|
||||
msg = logging.handlers.SysLogHandler.format(self, record)
|
||||
msg = self.binary_name + ' ' + msg
|
||||
return msg
|
||||
|
||||
|
||||
def _setup_logging_from_conf(project, version):
|
||||
log_root = getLogger(None).logger
|
||||
for handler in log_root.handlers:
|
||||
log_root.removeHandler(handler)
|
||||
|
||||
if CONF.use_syslog:
|
||||
facility = _find_facility_from_conf()
|
||||
# TODO(bogdando) use the format provided by RFCSysLogHandler
|
||||
# after existing syslog format deprecation in J
|
||||
if CONF.use_syslog_rfc_format:
|
||||
syslog = RFCSysLogHandler(address='/dev/log',
|
||||
facility=facility)
|
||||
else:
|
||||
syslog = logging.handlers.SysLogHandler(address='/dev/log',
|
||||
facility=facility)
|
||||
log_root.addHandler(syslog)
|
||||
|
||||
logpath = _get_log_file_path()
|
||||
if logpath:
|
||||
filelog = logging.handlers.WatchedFileHandler(logpath)
|
||||
log_root.addHandler(filelog)
|
||||
|
||||
if CONF.use_stderr:
|
||||
streamlog = ColorHandler()
|
||||
log_root.addHandler(streamlog)
|
||||
|
||||
elif not logpath:
|
||||
# pass sys.stdout as a positional argument
|
||||
# python2.6 calls the argument strm, in 2.7 it's stream
|
||||
streamlog = logging.StreamHandler(sys.stdout)
|
||||
log_root.addHandler(streamlog)
|
||||
|
||||
if CONF.publish_errors:
|
||||
handler = importutils.import_object(
|
||||
"fuel_agent.openstack.common.log_handler.PublishErrorsHandler",
|
||||
logging.ERROR)
|
||||
log_root.addHandler(handler)
|
||||
|
||||
datefmt = CONF.log_date_format
|
||||
for handler in log_root.handlers:
|
||||
# NOTE(alaski): CONF.log_format overrides everything currently. This
|
||||
# should be deprecated in favor of context aware formatting.
|
||||
if CONF.log_format:
|
||||
handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
|
||||
datefmt=datefmt))
|
||||
log_root.info('Deprecated: log_format is now deprecated and will '
|
||||
'be removed in the next release')
|
||||
else:
|
||||
handler.setFormatter(ContextFormatter(project=project,
|
||||
version=version,
|
||||
datefmt=datefmt))
|
||||
|
||||
if CONF.debug:
|
||||
log_root.setLevel(logging.DEBUG)
|
||||
elif CONF.verbose:
|
||||
log_root.setLevel(logging.INFO)
|
||||
else:
|
||||
log_root.setLevel(logging.WARNING)
|
||||
|
||||
for pair in CONF.default_log_levels:
|
||||
mod, _sep, level_name = pair.partition('=')
|
||||
logger = logging.getLogger(mod)
|
||||
# NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name
|
||||
# to integer code.
|
||||
if sys.version_info < (2, 7):
|
||||
level = logging.getLevelName(level_name)
|
||||
logger.setLevel(level)
|
||||
else:
|
||||
logger.setLevel(level_name)
|
||||
|
||||
|
||||
_loggers = {}
|
||||
|
||||
|
||||
def getLogger(name='unknown', version='unknown'):
|
||||
if name not in _loggers:
|
||||
_loggers[name] = ContextAdapter(logging.getLogger(name),
|
||||
name,
|
||||
version)
|
||||
return _loggers[name]
|
||||
|
||||
|
||||
def getLazyLogger(name='unknown', version='unknown'):
|
||||
"""Returns lazy logger.
|
||||
|
||||
Creates a pass-through logger that does not create the real logger
|
||||
until it is really needed and delegates all calls to the real logger
|
||||
once it is created.
|
||||
"""
|
||||
return LazyAdapter(name, version)
|
||||
|
||||
|
||||
class WritableLogger(object):
|
||||
"""A thin wrapper that responds to `write` and logs."""
|
||||
|
||||
def __init__(self, logger, level=logging.INFO):
|
||||
self.logger = logger
|
||||
self.level = level
|
||||
|
||||
def write(self, msg):
|
||||
self.logger.log(self.level, msg.rstrip())
|
||||
|
||||
|
||||
class ContextFormatter(logging.Formatter):
|
||||
"""A context.RequestContext aware formatter configured through flags.
|
||||
|
||||
The flags used to set format strings are: logging_context_format_string
|
||||
and logging_default_format_string. You can also specify
|
||||
logging_debug_format_suffix to append extra formatting if the log level is
|
||||
debug.
|
||||
|
||||
For information about what variables are available for the formatter see:
|
||||
http://docs.python.org/library/logging.html#formatter
|
||||
|
||||
If available, uses the context value stored in TLS - local.store.context
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""Initialize ContextFormatter instance
|
||||
|
||||
Takes additional keyword arguments which can be used in the message
|
||||
format string.
|
||||
|
||||
:keyword project: project name
|
||||
:type project: string
|
||||
:keyword version: project version
|
||||
:type version: string
|
||||
|
||||
"""
|
||||
|
||||
self.project = kwargs.pop('project', 'unknown')
|
||||
self.version = kwargs.pop('version', 'unknown')
|
||||
|
||||
logging.Formatter.__init__(self, *args, **kwargs)
|
||||
|
||||
def format(self, record):
|
||||
"""Uses contextstring if request_id is set, otherwise default."""
|
||||
|
||||
# store project info
|
||||
record.project = self.project
|
||||
record.version = self.version
|
||||
|
||||
# store request info
|
||||
context = getattr(local.store, 'context', None)
|
||||
if context:
|
||||
d = _dictify_context(context)
|
||||
for k, v in d.items():
|
||||
setattr(record, k, v)
|
||||
|
||||
# NOTE(sdague): default the fancier formatting params
|
||||
# to an empty string so we don't throw an exception if
|
||||
# they get used
|
||||
for key in ('instance', 'color', 'user_identity'):
|
||||
if key not in record.__dict__:
|
||||
record.__dict__[key] = ''
|
||||
|
||||
if record.__dict__.get('request_id'):
|
||||
self._fmt = CONF.logging_context_format_string
|
||||
else:
|
||||
self._fmt = CONF.logging_default_format_string
|
||||
|
||||
if (record.levelno == logging.DEBUG and
|
||||
CONF.logging_debug_format_suffix):
|
||||
self._fmt += " " + CONF.logging_debug_format_suffix
|
||||
|
||||
# Cache this on the record, Logger will respect our formatted copy
|
||||
if record.exc_info:
|
||||
record.exc_text = self.formatException(record.exc_info, record)
|
||||
return logging.Formatter.format(self, record)
|
||||
|
||||
def formatException(self, exc_info, record=None):
|
||||
"""Format exception output with CONF.logging_exception_prefix."""
|
||||
if not record:
|
||||
return logging.Formatter.formatException(self, exc_info)
|
||||
|
||||
stringbuffer = moves.StringIO()
|
||||
traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
|
||||
None, stringbuffer)
|
||||
lines = stringbuffer.getvalue().split('\n')
|
||||
stringbuffer.close()
|
||||
|
||||
if CONF.logging_exception_prefix.find('%(asctime)') != -1:
|
||||
record.asctime = self.formatTime(record, self.datefmt)
|
||||
|
||||
formatted_lines = []
|
||||
for line in lines:
|
||||
pl = CONF.logging_exception_prefix % record.__dict__
|
||||
fl = '%s%s' % (pl, line)
|
||||
formatted_lines.append(fl)
|
||||
return '\n'.join(formatted_lines)
|
||||
|
||||
|
||||
class ColorHandler(logging.StreamHandler):
|
||||
LEVEL_COLORS = {
|
||||
logging.DEBUG: '\033[00;32m', # GREEN
|
||||
logging.INFO: '\033[00;36m', # CYAN
|
||||
logging.AUDIT: '\033[01;36m', # BOLD CYAN
|
||||
logging.WARN: '\033[01;33m', # BOLD YELLOW
|
||||
logging.ERROR: '\033[01;31m', # BOLD RED
|
||||
logging.CRITICAL: '\033[01;31m', # BOLD RED
|
||||
}
|
||||
|
||||
def format(self, record):
|
||||
record.color = self.LEVEL_COLORS[record.levelno]
|
||||
return logging.StreamHandler.format(self, record)
|
||||
|
||||
|
||||
class DeprecatedConfig(Exception):
|
||||
message = _("Fatal call to deprecated config: %(msg)s")
|
||||
|
||||
def __init__(self, msg):
|
||||
super(Exception, self).__init__(self.message % dict(msg=msg))
|
|
@ -1,272 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
System-level utilities and helper functions.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import logging as stdlib_logging
|
||||
import os
|
||||
import random
|
||||
import shlex
|
||||
import signal
|
||||
|
||||
from eventlet.green import subprocess
|
||||
from eventlet import greenthread
|
||||
import six
|
||||
|
||||
from fuel_agent.openstack.common.gettextutils import _
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InvalidArgumentError(Exception):
|
||||
def __init__(self, message=None):
|
||||
super(InvalidArgumentError, self).__init__(message)
|
||||
|
||||
|
||||
class UnknownArgumentError(Exception):
|
||||
def __init__(self, message=None):
|
||||
super(UnknownArgumentError, self).__init__(message)
|
||||
|
||||
|
||||
class ProcessExecutionError(Exception):
|
||||
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
|
||||
description=None):
|
||||
self.exit_code = exit_code
|
||||
self.stderr = stderr
|
||||
self.stdout = stdout
|
||||
self.cmd = cmd
|
||||
self.description = description
|
||||
|
||||
if description is None:
|
||||
description = _("Unexpected error while running command.")
|
||||
if exit_code is None:
|
||||
exit_code = '-'
|
||||
message = _('%(description)s\n'
|
||||
'Command: %(cmd)s\n'
|
||||
'Exit code: %(exit_code)s\n'
|
||||
'Stdout: %(stdout)r\n'
|
||||
'Stderr: %(stderr)r') % {'description': description,
|
||||
'cmd': cmd,
|
||||
'exit_code': exit_code,
|
||||
'stdout': stdout,
|
||||
'stderr': stderr}
|
||||
super(ProcessExecutionError, self).__init__(message)
|
||||
|
||||
|
||||
class NoRootWrapSpecified(Exception):
|
||||
def __init__(self, message=None):
|
||||
super(NoRootWrapSpecified, self).__init__(message)
|
||||
|
||||
|
||||
def _subprocess_setup():
|
||||
# Python installs a SIGPIPE handler by default. This is usually not what
|
||||
# non-Python subprocesses expect.
|
||||
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
||||
|
||||
|
||||
def execute(*cmd, **kwargs):
|
||||
"""Helper method to shell out and execute a command through subprocess.
|
||||
|
||||
Allows optional retry.
|
||||
|
||||
:param cmd: Passed to subprocess.Popen.
|
||||
:type cmd: string
|
||||
:param process_input: Send to opened process.
|
||||
:type process_input: string
|
||||
:param env_variables: Environment variables and their values that
|
||||
will be set for the process.
|
||||
:type env_variables: dict
|
||||
:param check_exit_code: Single bool, int, or list of allowed exit
|
||||
codes. Defaults to [0]. Raise
|
||||
:class:`ProcessExecutionError` unless
|
||||
program exits with one of these code.
|
||||
:type check_exit_code: boolean, int, or [int]
|
||||
:param delay_on_retry: True | False. Defaults to True. If set to True,
|
||||
wait a short amount of time before retrying.
|
||||
:type delay_on_retry: boolean
|
||||
:param attempts: How many times to retry cmd.
|
||||
:type attempts: int
|
||||
:param run_as_root: True | False. Defaults to False. If set to True,
|
||||
the command is prefixed by the command specified
|
||||
in the root_helper kwarg.
|
||||
:type run_as_root: boolean
|
||||
:param root_helper: command to prefix to commands called with
|
||||
run_as_root=True
|
||||
:type root_helper: string
|
||||
:param shell: whether or not there should be a shell used to
|
||||
execute this command. Defaults to false.
|
||||
:type shell: boolean
|
||||
:param loglevel: log level for execute commands.
|
||||
:type loglevel: int. (Should be stdlib_logging.DEBUG or
|
||||
stdlib_logging.INFO)
|
||||
:returns: (stdout, stderr) from process execution
|
||||
:raises: :class:`UnknownArgumentError` on
|
||||
receiving unknown arguments
|
||||
:raises: :class:`ProcessExecutionError`
|
||||
"""
|
||||
|
||||
process_input = kwargs.pop('process_input', None)
|
||||
env_variables = kwargs.pop('env_variables', None)
|
||||
check_exit_code = kwargs.pop('check_exit_code', [0])
|
||||
ignore_exit_code = False
|
||||
delay_on_retry = kwargs.pop('delay_on_retry', True)
|
||||
attempts = kwargs.pop('attempts', 1)
|
||||
run_as_root = kwargs.pop('run_as_root', False)
|
||||
root_helper = kwargs.pop('root_helper', '')
|
||||
shell = kwargs.pop('shell', False)
|
||||
loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
|
||||
|
||||
if isinstance(check_exit_code, bool):
|
||||
ignore_exit_code = not check_exit_code
|
||||
check_exit_code = [0]
|
||||
elif isinstance(check_exit_code, int):
|
||||
check_exit_code = [check_exit_code]
|
||||
|
||||
if kwargs:
|
||||
raise UnknownArgumentError(_('Got unknown keyword args '
|
||||
'to utils.execute: %r') % kwargs)
|
||||
|
||||
if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
|
||||
if not root_helper:
|
||||
raise NoRootWrapSpecified(
|
||||
message=_('Command requested root, but did not '
|
||||
'specify a root helper.'))
|
||||
cmd = shlex.split(root_helper) + list(cmd)
|
||||
|
||||
cmd = map(str, cmd)
|
||||
|
||||
while attempts > 0:
|
||||
attempts -= 1
|
||||
try:
|
||||
LOG.log(loglevel, 'Running cmd (subprocess): %s',
|
||||
' '.join(logging.mask_password(cmd)))
|
||||
_PIPE = subprocess.PIPE # pylint: disable=E1101
|
||||
|
||||
if os.name == 'nt':
|
||||
preexec_fn = None
|
||||
close_fds = False
|
||||
else:
|
||||
preexec_fn = _subprocess_setup
|
||||
close_fds = True
|
||||
|
||||
obj = subprocess.Popen(cmd,
|
||||
stdin=_PIPE,
|
||||
stdout=_PIPE,
|
||||
stderr=_PIPE,
|
||||
close_fds=close_fds,
|
||||
preexec_fn=preexec_fn,
|
||||
shell=shell,
|
||||
env=env_variables)
|
||||
result = None
|
||||
for _i in six.moves.range(20):
|
||||
# NOTE(russellb) 20 is an arbitrary number of retries to
|
||||
# prevent any chance of looping forever here.
|
||||
try:
|
||||
if process_input is not None:
|
||||
result = obj.communicate(process_input)
|
||||
else:
|
||||
result = obj.communicate()
|
||||
except OSError as e:
|
||||
if e.errno in (errno.EAGAIN, errno.EINTR):
|
||||
continue
|
||||
raise
|
||||
break
|
||||
obj.stdin.close() # pylint: disable=E1101
|
||||
_returncode = obj.returncode # pylint: disable=E1101
|
||||
LOG.log(loglevel, 'Result was %s' % _returncode)
|
||||
if not ignore_exit_code and _returncode not in check_exit_code:
|
||||
(stdout, stderr) = result
|
||||
raise ProcessExecutionError(exit_code=_returncode,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
cmd=' '.join(cmd))
|
||||
return result
|
||||
except ProcessExecutionError:
|
||||
if not attempts:
|
||||
raise
|
||||
else:
|
||||
LOG.log(loglevel, '%r failed. Retrying.', cmd)
|
||||
if delay_on_retry:
|
||||
greenthread.sleep(random.randint(20, 200) / 100.0)
|
||||
finally:
|
||||
# NOTE(termie): this appears to be necessary to let the subprocess
|
||||
# call clean something up in between calls, without
|
||||
# it two execute calls in a row hangs the second one
|
||||
greenthread.sleep(0)
|
||||
|
||||
|
||||
def trycmd(*args, **kwargs):
|
||||
"""A wrapper around execute() to more easily handle warnings and errors.
|
||||
|
||||
Returns an (out, err) tuple of strings containing the output of
|
||||
the command's stdout and stderr. If 'err' is not empty then the
|
||||
command can be considered to have failed.
|
||||
|
||||
:discard_warnings True | False. Defaults to False. If set to True,
|
||||
then for succeeding commands, stderr is cleared
|
||||
|
||||
"""
|
||||
discard_warnings = kwargs.pop('discard_warnings', False)
|
||||
|
||||
try:
|
||||
out, err = execute(*args, **kwargs)
|
||||
failed = False
|
||||
except ProcessExecutionError as exn:
|
||||
out, err = '', six.text_type(exn)
|
||||
failed = True
|
||||
|
||||
if not failed and discard_warnings and err:
|
||||
# Handle commands that output to stderr but otherwise succeed
|
||||
err = ''
|
||||
|
||||
return out, err
|
||||
|
||||
|
||||
def ssh_execute(ssh, cmd, process_input=None,
|
||||
addl_env=None, check_exit_code=True):
|
||||
LOG.debug('Running cmd (SSH): %s', cmd)
|
||||
if addl_env:
|
||||
raise InvalidArgumentError(_('Environment not supported over SSH'))
|
||||
|
||||
if process_input:
|
||||
# This is (probably) fixable if we need it...
|
||||
raise InvalidArgumentError(_('process_input not supported over SSH'))
|
||||
|
||||
stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
|
||||
channel = stdout_stream.channel
|
||||
|
||||
# NOTE(justinsb): This seems suspicious...
|
||||
# ...other SSH clients have buffering issues with this approach
|
||||
stdout = stdout_stream.read()
|
||||
stderr = stderr_stream.read()
|
||||
stdin_stream.close()
|
||||
|
||||
exit_status = channel.recv_exit_status()
|
||||
|
||||
# exit_status == -1 if no exit code was returned
|
||||
if exit_status != -1:
|
||||
LOG.debug('Result was %s' % exit_status)
|
||||
if check_exit_code and exit_status != 0:
|
||||
raise ProcessExecutionError(exit_code=exit_status,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
cmd=cmd)
|
||||
|
||||
return (stdout, stderr)
|
|
@ -1,239 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
System-level utilities and helper functions.
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
import six
|
||||
|
||||
from fuel_agent.openstack.common.gettextutils import _
|
||||
|
||||
|
||||
UNIT_PREFIX_EXPONENT = {
|
||||
'k': 1,
|
||||
'K': 1,
|
||||
'Ki': 1,
|
||||
'M': 2,
|
||||
'Mi': 2,
|
||||
'G': 3,
|
||||
'Gi': 3,
|
||||
'T': 4,
|
||||
'Ti': 4,
|
||||
}
|
||||
UNIT_SYSTEM_INFO = {
|
||||
'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
|
||||
'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
|
||||
}
|
||||
|
||||
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
|
||||
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
|
||||
|
||||
SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
|
||||
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
|
||||
|
||||
|
||||
def int_from_bool_as_string(subject):
|
||||
"""Interpret a string as a boolean and return either 1 or 0.
|
||||
|
||||
Any string value in:
|
||||
|
||||
('True', 'true', 'On', 'on', '1')
|
||||
|
||||
is interpreted as a boolean True.
|
||||
|
||||
Useful for JSON-decoded stuff and config file parsing
|
||||
"""
|
||||
return bool_from_string(subject) and 1 or 0
|
||||
|
||||
|
||||
def bool_from_string(subject, strict=False, default=False):
|
||||
"""Interpret a string as a boolean.
|
||||
|
||||
A case-insensitive match is performed such that strings matching 't',
|
||||
'true', 'on', 'y', 'yes', or '1' are considered True and, when
|
||||
`strict=False`, anything else returns the value specified by 'default'.
|
||||
|
||||
Useful for JSON-decoded stuff and config file parsing.
|
||||
|
||||
If `strict=True`, unrecognized values, including None, will raise a
|
||||
ValueError which is useful when parsing values passed in from an API call.
|
||||
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
|
||||
"""
|
||||
if not isinstance(subject, six.string_types):
|
||||
subject = six.text_type(subject)
|
||||
|
||||
lowered = subject.strip().lower()
|
||||
|
||||
if lowered in TRUE_STRINGS:
|
||||
return True
|
||||
elif lowered in FALSE_STRINGS:
|
||||
return False
|
||||
elif strict:
|
||||
acceptable = ', '.join(
|
||||
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
|
||||
msg = _("Unrecognized value '%(val)s', acceptable values are:"
|
||||
" %(acceptable)s") % {'val': subject,
|
||||
'acceptable': acceptable}
|
||||
raise ValueError(msg)
|
||||
else:
|
||||
return default
|
||||
|
||||
|
||||
def safe_decode(text, incoming=None, errors='strict'):
|
||||
"""Decodes incoming text/bytes string using `incoming` if they're not
|
||||
already unicode.
|
||||
|
||||
:param incoming: Text's current encoding
|
||||
:param errors: Errors handling policy. See here for valid
|
||||
values http://docs.python.org/2/library/codecs.html
|
||||
:returns: text or a unicode `incoming` encoded
|
||||
representation of it.
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||
raise TypeError("%s can't be decoded" % type(text))
|
||||
|
||||
if isinstance(text, six.text_type):
|
||||
return text
|
||||
|
||||
if not incoming:
|
||||
incoming = (sys.stdin.encoding or
|
||||
sys.getdefaultencoding())
|
||||
|
||||
try:
|
||||
return text.decode(incoming, errors)
|
||||
except UnicodeDecodeError:
|
||||
# Note(flaper87) If we get here, it means that
|
||||
# sys.stdin.encoding / sys.getdefaultencoding
|
||||
# didn't return a suitable encoding to decode
|
||||
# text. This happens mostly when global LANG
|
||||
# var is not set correctly and there's no
|
||||
# default encoding. In this case, most likely
|
||||
# python will use ASCII or ANSI encoders as
|
||||
# default encodings but they won't be capable
|
||||
# of decoding non-ASCII characters.
|
||||
#
|
||||
# Also, UTF-8 is being used since it's an ASCII
|
||||
# extension.
|
||||
return text.decode('utf-8', errors)
|
||||
|
||||
|
||||
def safe_encode(text, incoming=None,
|
||||
encoding='utf-8', errors='strict'):
|
||||
"""Encodes incoming text/bytes string using `encoding`.
|
||||
|
||||
If incoming is not specified, text is expected to be encoded with
|
||||
current python's default encoding. (`sys.getdefaultencoding`)
|
||||
|
||||
:param incoming: Text's current encoding
|
||||
:param encoding: Expected encoding for text (Default UTF-8)
|
||||
:param errors: Errors handling policy. See here for valid
|
||||
values http://docs.python.org/2/library/codecs.html
|
||||
:returns: text or a bytestring `encoding` encoded
|
||||
representation of it.
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||
raise TypeError("%s can't be encoded" % type(text))
|
||||
|
||||
if not incoming:
|
||||
incoming = (sys.stdin.encoding or
|
||||
sys.getdefaultencoding())
|
||||
|
||||
if isinstance(text, six.text_type):
|
||||
return text.encode(encoding, errors)
|
||||
elif text and encoding != incoming:
|
||||
# Decode text before encoding it with `encoding`
|
||||
text = safe_decode(text, incoming, errors)
|
||||
return text.encode(encoding, errors)
|
||||
else:
|
||||
return text
|
||||
|
||||
|
||||
def string_to_bytes(text, unit_system='IEC', return_int=False):
|
||||
"""Converts a string into an float representation of bytes.
|
||||
|
||||
The units supported for IEC ::
|
||||
|
||||
Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
|
||||
KB, KiB, MB, MiB, GB, GiB, TB, TiB
|
||||
|
||||
The units supported for SI ::
|
||||
|
||||
kb(it), Mb(it), Gb(it), Tb(it)
|
||||
kB, MB, GB, TB
|
||||
|
||||
Note that the SI unit system does not support capital letter 'K'
|
||||
|
||||
:param text: String input for bytes size conversion.
|
||||
:param unit_system: Unit system for byte size conversion.
|
||||
:param return_int: If True, returns integer representation of text
|
||||
in bytes. (default: decimal)
|
||||
:returns: Numerical representation of text in bytes.
|
||||
:raises ValueError: If text has an invalid value.
|
||||
|
||||
"""
|
||||
try:
|
||||
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
|
||||
except KeyError:
|
||||
msg = _('Invalid unit system: "%s"') % unit_system
|
||||
raise ValueError(msg)
|
||||
match = reg_ex.match(text)
|
||||
if match:
|
||||
magnitude = float(match.group(1))
|
||||
unit_prefix = match.group(2)
|
||||
if match.group(3) in ['b', 'bit']:
|
||||
magnitude /= 8
|
||||
else:
|
||||
msg = _('Invalid string format: %s') % text
|
||||
raise ValueError(msg)
|
||||
if not unit_prefix:
|
||||
res = magnitude
|
||||
else:
|
||||
res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
|
||||
if return_int:
|
||||
return int(math.ceil(res))
|
||||
return res
|
||||
|
||||
|
||||
def to_slug(value, incoming=None, errors="strict"):
|
||||
"""Normalize string.
|
||||
|
||||
Convert to lowercase, remove non-word characters, and convert spaces
|
||||
to hyphens.
|
||||
|
||||
Inspired by Django's `slugify` filter.
|
||||
|
||||
:param value: Text to slugify
|
||||
:param incoming: Text's current encoding
|
||||
:param errors: Errors handling policy. See here for valid
|
||||
values http://docs.python.org/2/library/codecs.html
|
||||
:returns: slugified unicode representation of `value`
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
value = safe_decode(value, incoming, errors)
|
||||
# NOTE(aababilov): no need to use safe_(encode|decode) here:
|
||||
# encodings are always "ascii", error handling is always "ignore"
|
||||
# and types are always known (first: unicode; second: str)
|
||||
value = unicodedata.normalize("NFKD", value).encode(
|
||||
"ascii", "ignore").decode("ascii")
|
||||
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
|
||||
return SLUGIFY_HYPHENATE_RE.sub("-", value)
|
|
@ -1,210 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Time related utilities and helper functions.
|
||||
"""
|
||||
|
||||
import calendar
|
||||
import datetime
|
||||
import time
|
||||
|
||||
import iso8601
|
||||
import six
|
||||
|
||||
|
||||
# ISO 8601 extended time format with microseconds
|
||||
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
||||
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
||||
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
|
||||
|
||||
|
||||
def isotime(at=None, subsecond=False):
|
||||
"""Stringify time in ISO 8601 format."""
|
||||
if not at:
|
||||
at = utcnow()
|
||||
st = at.strftime(_ISO8601_TIME_FORMAT
|
||||
if not subsecond
|
||||
else _ISO8601_TIME_FORMAT_SUBSECOND)
|
||||
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
||||
st += ('Z' if tz == 'UTC' else tz)
|
||||
return st
|
||||
|
||||
|
||||
def parse_isotime(timestr):
|
||||
"""Parse time from ISO 8601 format."""
|
||||
try:
|
||||
return iso8601.parse_date(timestr)
|
||||
except iso8601.ParseError as e:
|
||||
raise ValueError(six.text_type(e))
|
||||
except TypeError as e:
|
||||
raise ValueError(six.text_type(e))
|
||||
|
||||
|
||||
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
|
||||
"""Returns formatted utcnow."""
|
||||
if not at:
|
||||
at = utcnow()
|
||||
return at.strftime(fmt)
|
||||
|
||||
|
||||
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
|
||||
"""Turn a formatted time back into a datetime."""
|
||||
return datetime.datetime.strptime(timestr, fmt)
|
||||
|
||||
|
||||
def normalize_time(timestamp):
|
||||
"""Normalize time in arbitrary timezone to UTC naive object."""
|
||||
offset = timestamp.utcoffset()
|
||||
if offset is None:
|
||||
return timestamp
|
||||
return timestamp.replace(tzinfo=None) - offset
|
||||
|
||||
|
||||
def is_older_than(before, seconds):
|
||||
"""Return True if before is older than seconds."""
|
||||
if isinstance(before, six.string_types):
|
||||
before = parse_strtime(before).replace(tzinfo=None)
|
||||
else:
|
||||
before = before.replace(tzinfo=None)
|
||||
|
||||
return utcnow() - before > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def is_newer_than(after, seconds):
|
||||
"""Return True if after is newer than seconds."""
|
||||
if isinstance(after, six.string_types):
|
||||
after = parse_strtime(after).replace(tzinfo=None)
|
||||
else:
|
||||
after = after.replace(tzinfo=None)
|
||||
|
||||
return after - utcnow() > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def utcnow_ts():
|
||||
"""Timestamp version of our utcnow function."""
|
||||
if utcnow.override_time is None:
|
||||
# NOTE(kgriffs): This is several times faster
|
||||
# than going through calendar.timegm(...)
|
||||
return int(time.time())
|
||||
|
||||
return calendar.timegm(utcnow().timetuple())
|
||||
|
||||
|
||||
def utcnow():
|
||||
"""Overridable version of utils.utcnow."""
|
||||
if utcnow.override_time:
|
||||
try:
|
||||
return utcnow.override_time.pop(0)
|
||||
except AttributeError:
|
||||
return utcnow.override_time
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def iso8601_from_timestamp(timestamp):
|
||||
"""Returns a iso8601 formatted date from timestamp."""
|
||||
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
|
||||
|
||||
|
||||
utcnow.override_time = None
|
||||
|
||||
|
||||
def set_time_override(override_time=None):
|
||||
"""Overrides utils.utcnow.
|
||||
|
||||
Make it return a constant time or a list thereof, one at a time.
|
||||
|
||||
:param override_time: datetime instance or list thereof. If not
|
||||
given, defaults to the current UTC time.
|
||||
"""
|
||||
utcnow.override_time = override_time or datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def advance_time_delta(timedelta):
|
||||
"""Advance overridden time using a datetime.timedelta."""
|
||||
assert(not utcnow.override_time is None)
|
||||
try:
|
||||
for dt in utcnow.override_time:
|
||||
dt += timedelta
|
||||
except TypeError:
|
||||
utcnow.override_time += timedelta
|
||||
|
||||
|
||||
def advance_time_seconds(seconds):
|
||||
"""Advance overridden time by seconds."""
|
||||
advance_time_delta(datetime.timedelta(0, seconds))
|
||||
|
||||
|
||||
def clear_time_override():
|
||||
"""Remove the overridden time."""
|
||||
utcnow.override_time = None
|
||||
|
||||
|
||||
def marshall_now(now=None):
|
||||
"""Make an rpc-safe datetime with microseconds.
|
||||
|
||||
Note: tzinfo is stripped, but not required for relative times.
|
||||
"""
|
||||
if not now:
|
||||
now = utcnow()
|
||||
return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
|
||||
minute=now.minute, second=now.second,
|
||||
microsecond=now.microsecond)
|
||||
|
||||
|
||||
def unmarshall_time(tyme):
|
||||
"""Unmarshall a datetime dict."""
|
||||
return datetime.datetime(day=tyme['day'],
|
||||
month=tyme['month'],
|
||||
year=tyme['year'],
|
||||
hour=tyme['hour'],
|
||||
minute=tyme['minute'],
|
||||
second=tyme['second'],
|
||||
microsecond=tyme['microsecond'])
|
||||
|
||||
|
||||
def delta_seconds(before, after):
|
||||
"""Return the difference between two timing objects.
|
||||
|
||||
Compute the difference in seconds between two date, time, or
|
||||
datetime objects (as a float, to microsecond resolution).
|
||||
"""
|
||||
delta = after - before
|
||||
return total_seconds(delta)
|
||||
|
||||
|
||||
def total_seconds(delta):
|
||||
"""Return the total seconds of datetime.timedelta object.
|
||||
|
||||
Compute total seconds of datetime.timedelta, datetime.timedelta
|
||||
doesn't have method total_seconds in Python2.6, calculate it manually.
|
||||
"""
|
||||
try:
|
||||
return delta.total_seconds()
|
||||
except AttributeError:
|
||||
return ((delta.days * 24 * 3600) + delta.seconds +
|
||||
float(delta.microseconds) / (10 ** 6))
|
||||
|
||||
|
||||
def is_soon(dt, window):
|
||||
"""Determines if time is going to happen in the next window seconds.
|
||||
|
||||
:param dt: the time
|
||||
:param window: minimum seconds to remain to consider the time not soon
|
||||
|
||||
:return: True if expiration is within the given duration
|
||||
"""
|
||||
soon = (utcnow() + datetime.timedelta(seconds=window))
|
||||
return normalize_time(dt) <= soon
|
|
@ -1,119 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
import zlib
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import artifact as au
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class TestTarget(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestTarget, self).setUp()
|
||||
self.tgt = au.Target()
|
||||
|
||||
def test_target_next(self):
|
||||
self.assertRaises(StopIteration, self.tgt.next)
|
||||
|
||||
@mock.patch('os.fsync')
|
||||
@mock.patch.object(au.Target, '__iter__')
|
||||
def test_target_target(self, mock_iter, mock_os_sync):
|
||||
mock_iter.return_value = iter(['chunk1', 'chunk2', 'chunk3'])
|
||||
m = mock.mock_open()
|
||||
with mock.patch('six.moves.builtins.open', m):
|
||||
self.tgt.target()
|
||||
|
||||
mock_write_expected_calls = [mock.call('chunk1'), mock.call('chunk2'),
|
||||
mock.call('chunk3')]
|
||||
file_handle = m()
|
||||
self.assertEqual(mock_write_expected_calls,
|
||||
file_handle.write.call_args_list)
|
||||
file_handle.flush.assert_called_once_with()
|
||||
|
||||
|
||||
class TestLocalFile(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestLocalFile, self).setUp()
|
||||
self.lf = au.LocalFile('/dev/null')
|
||||
|
||||
def test_localfile_next(self):
|
||||
self.lf.fileobj = mock.Mock()
|
||||
self.lf.fileobj.read.side_effect = ['some_data', 'another_data']
|
||||
self.assertEqual('some_data', self.lf.next())
|
||||
self.assertEqual('another_data', self.lf.next())
|
||||
self.assertRaises(StopIteration, self.lf.next)
|
||||
|
||||
|
||||
class TestHttpUrl(test_base.BaseTestCase):
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
def test_httpurl_init_ok(self, mock_req):
|
||||
mock_req.return_value = mock.Mock(headers={'content-length': 123})
|
||||
httpurl = au.HttpUrl('fake_url')
|
||||
self.assertEqual(123, httpurl.length)
|
||||
mock_req.assert_called_once_with('fake_url')
|
||||
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
def test_httpurl_init_invalid_content_length(self, mock_req):
|
||||
mock_req.return_value = mock.Mock(headers={'content-length':
|
||||
'invalid'})
|
||||
self.assertRaises(errors.HttpUrlInvalidContentLength, au.HttpUrl,
|
||||
'fake_url')
|
||||
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
def test_httpurl_next_ok(self, mock_req):
|
||||
content = ['fake content #1', 'fake content #2']
|
||||
req_mock = mock.Mock(headers={'content-length': 30})
|
||||
req_mock.raw.read.side_effect = content
|
||||
mock_req.return_value = req_mock
|
||||
httpurl = au.HttpUrl('fake_url')
|
||||
for data in enumerate(httpurl):
|
||||
self.assertEqual(content[data[0]], data[1])
|
||||
|
||||
|
||||
class TestGunzipStream(test_base.BaseTestCase):
|
||||
def test_gunzip_stream_next(self):
|
||||
content = ['fake content #1']
|
||||
compressed_stream = [zlib.compress(data) for data in content]
|
||||
gunzip_stream = au.GunzipStream(compressed_stream)
|
||||
for data in enumerate(gunzip_stream):
|
||||
self.assertEqual(content[data[0]], data[1])
|
||||
|
||||
|
||||
class TestChain(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestChain, self).setUp()
|
||||
self.chain = au.Chain()
|
||||
|
||||
def test_append(self):
|
||||
self.assertEqual(0, len(self.chain.processors))
|
||||
self.chain.append('fake_processor')
|
||||
self.assertIn('fake_processor', self.chain.processors)
|
||||
self.assertEqual(1, len(self.chain.processors))
|
||||
|
||||
def test_process(self):
|
||||
self.chain.processors.append('fake_uri')
|
||||
fake_processor = mock.Mock(spec=au.Target)
|
||||
self.chain.processors.append(fake_processor)
|
||||
self.chain.processors.append('fake_target')
|
||||
self.chain.process()
|
||||
expected_calls = [mock.call('fake_uri')]
|
||||
self.assertEqual(expected_calls, fake_processor.call_args_list)
|
|
@ -1,489 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import testtools
|
||||
|
||||
import mock
|
||||
from oslo.config import cfg
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import build as bu
|
||||
from fuel_agent.utils import hardware as hu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class BuildUtilsTestCase(testtools.TestCase):
|
||||
|
||||
_fake_ubuntu_release = '''
|
||||
Origin: TestOrigin
|
||||
Label: TestLabel
|
||||
Archive: test-archive
|
||||
Codename: testcodename
|
||||
'''
|
||||
|
||||
def setUp(self):
|
||||
super(BuildUtilsTestCase, self).setUp()
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_debootstrap(self, mock_exec):
|
||||
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', attempts=2)
|
||||
mock_exec.assert_called_once_with('debootstrap', '--verbose',
|
||||
'--no-check-gpg', '--arch=arch',
|
||||
'suite', 'chroot', 'uri', attempts=2)
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_debootstrap_eatmydata(self, mock_exec):
|
||||
bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True,
|
||||
attempts=2)
|
||||
mock_exec.assert_called_once_with('debootstrap', '--verbose',
|
||||
'--no-check-gpg', '--arch=arch',
|
||||
'--include=eatmydata', 'suite',
|
||||
'chroot', 'uri', attempts=2)
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_apt_get(self, mock_exec):
|
||||
bu.run_apt_get('chroot', ['package1', 'package2'], attempts=2)
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('chroot', 'chroot', 'apt-get', '-y', 'update',
|
||||
attempts=2),
|
||||
mock.call('chroot', 'chroot', 'apt-get', '-y', 'install',
|
||||
'package1 package2', attempts=2)]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'execute', return_value=(None, None))
|
||||
def test_run_apt_get_eatmydata(self, mock_exec):
|
||||
bu.run_apt_get('chroot', ['package1', 'package2'], eatmydata=True,
|
||||
attempts=2)
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('chroot', 'chroot', 'apt-get', '-y', 'update',
|
||||
attempts=2),
|
||||
mock.call('chroot', 'chroot', 'eatmydata', 'apt-get', '-y',
|
||||
'install', 'package1 package2', attempts=2)]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'fchmod')
|
||||
@mock.patch.object(os, 'makedirs')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_suppress_services_start(self, mock_path, mock_mkdir, mock_fchmod):
|
||||
mock_path.join.return_value = 'fake_path'
|
||||
mock_path.exists.return_value = False
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
file_handle_mock.fileno.return_value = 'fake_fileno'
|
||||
bu.suppress_services_start('chroot')
|
||||
mock_open.assert_called_once_with('fake_path', 'w')
|
||||
expected = '#!/bin/sh\n# prevent any service from being started\n'\
|
||||
'exit 101\n'
|
||||
file_handle_mock.write.assert_called_once_with(expected)
|
||||
mock_fchmod.assert_called_once_with('fake_fileno', 0o755)
|
||||
mock_mkdir.assert_called_once_with('fake_path')
|
||||
|
||||
@mock.patch.object(os, 'fchmod')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_suppress_services_start_nomkdir(self, mock_path, mock_fchmod):
|
||||
mock_path.join.return_value = 'fake_path'
|
||||
mock_path.exists.return_value = True
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
file_handle_mock.fileno.return_value = 'fake_fileno'
|
||||
bu.suppress_services_start('chroot')
|
||||
mock_open.assert_called_once_with('fake_path', 'w')
|
||||
expected = '#!/bin/sh\n# prevent any service from being started\n'\
|
||||
'exit 101\n'
|
||||
file_handle_mock.write.assert_called_once_with(expected)
|
||||
mock_fchmod.assert_called_once_with('fake_fileno', 0o755)
|
||||
|
||||
@mock.patch.object(shutil, 'rmtree')
|
||||
@mock.patch.object(os, 'makedirs')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_clean_dirs(self, mock_path, mock_mkdir, mock_rmtree):
|
||||
mock_path.isdir.return_value = True
|
||||
dirs = ['dir1', 'dir2', 'dir3']
|
||||
mock_path.join.side_effect = dirs
|
||||
bu.clean_dirs('chroot', dirs)
|
||||
for m in (mock_rmtree, mock_mkdir):
|
||||
self.assertEqual([mock.call(d) for d in dirs], m.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_clean_dirs_not_isdir(self, mock_path):
|
||||
mock_path.isdir.return_value = False
|
||||
dirs = ['dir1', 'dir2', 'dir3']
|
||||
mock_path.join.side_effect = dirs
|
||||
bu.clean_dirs('chroot', dirs)
|
||||
self.assertEqual([mock.call('chroot', d) for d in dirs],
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'remove')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_remove_files(self, mock_path, mock_remove):
|
||||
mock_path.exists.return_value = True
|
||||
files = ['file1', 'file2', 'dir3']
|
||||
mock_path.join.side_effect = files
|
||||
bu.remove_files('chroot', files)
|
||||
self.assertEqual([mock.call(f) for f in files],
|
||||
mock_remove.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_remove_files_not_exists(self, mock_path):
|
||||
mock_path.exists.return_value = False
|
||||
files = ['file1', 'file2', 'dir3']
|
||||
mock_path.join.side_effect = files
|
||||
bu.remove_files('chroot', files)
|
||||
self.assertEqual([mock.call('chroot', f) for f in files],
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(bu, 'remove_files')
|
||||
@mock.patch.object(bu, 'clean_dirs')
|
||||
def test_clean_apt_settings(self, mock_dirs, mock_files):
|
||||
bu.clean_apt_settings('chroot', 'unsigned', 'force_ipv4')
|
||||
mock_dirs.assert_called_once_with(
|
||||
'chroot', ['etc/apt/preferences.d', 'etc/apt/sources.list.d'])
|
||||
mock_files.assert_called_once_with(
|
||||
'chroot', ['etc/apt/sources.list', 'etc/apt/preferences',
|
||||
'etc/apt/apt.conf.d/%s' % 'force_ipv4',
|
||||
'etc/apt/apt.conf.d/%s' % 'unsigned'])
|
||||
|
||||
@mock.patch('fuel_agent.utils.build.open',
|
||||
create=True, new_callable=mock.mock_open)
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch.object(bu, 'clean_apt_settings')
|
||||
@mock.patch.object(bu, 'remove_files')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_do_post_inst(self, mock_exec, mock_files, mock_clean, mock_path,
|
||||
mock_open):
|
||||
mock_path.join.return_value = 'fake_path'
|
||||
bu.do_post_inst('chroot')
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
file_handle_mock.write.assert_called_once_with('manual\n')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('sed', '-i', 's%root:[\*,\!]%root:$6$IInX3Cqo$5xytL1VZb'
|
||||
'ZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYclwHqMq9e3Tg8rvQx'
|
||||
'hxSlBXP1DZhdUamxdOBXK0.%', 'fake_path'),
|
||||
mock.call('chroot', 'chroot', 'update-rc.d', 'puppet', 'disable')]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
mock_files.assert_called_once_with('chroot', ['usr/sbin/policy-rc.d'])
|
||||
mock_clean.assert_called_once_with('chroot')
|
||||
mock_path_join_expected_calls = [
|
||||
mock.call('chroot', 'etc/shadow'),
|
||||
mock.call('chroot', 'etc/init/mcollective.override')]
|
||||
self.assertEqual(mock_path_join_expected_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch('fuel_agent.utils.build.open',
|
||||
create=True, new_callable=mock.mock_open)
|
||||
@mock.patch('fuel_agent.utils.build.time.sleep')
|
||||
@mock.patch.object(os, 'kill')
|
||||
@mock.patch.object(os, 'readlink', return_value='chroot')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_stop_chrooted_processes(self, mock_exec, mock_link,
|
||||
mock_kill, mock_sleep, mock_open):
|
||||
mock_exec.side_effect = [
|
||||
('kernel 951 1641 1700 1920 3210 4104', ''),
|
||||
('kernel 951 1641 1700', ''),
|
||||
('', '')]
|
||||
mock_exec_expected_calls = \
|
||||
[mock.call('fuser', '-v', 'chroot', check_exit_code=False)] * 3
|
||||
|
||||
bu.stop_chrooted_processes('chroot', signal=signal.SIGTERM)
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
expected_mock_link_calls = [
|
||||
mock.call('/proc/951/root'),
|
||||
mock.call('/proc/1641/root'),
|
||||
mock.call('/proc/1700/root'),
|
||||
mock.call('/proc/1920/root'),
|
||||
mock.call('/proc/3210/root'),
|
||||
mock.call('/proc/4104/root'),
|
||||
mock.call('/proc/951/root'),
|
||||
mock.call('/proc/1641/root'),
|
||||
mock.call('/proc/1700/root')]
|
||||
expected_mock_kill_calls = [
|
||||
mock.call(951, signal.SIGTERM),
|
||||
mock.call(1641, signal.SIGTERM),
|
||||
mock.call(1700, signal.SIGTERM),
|
||||
mock.call(1920, signal.SIGTERM),
|
||||
mock.call(3210, signal.SIGTERM),
|
||||
mock.call(4104, signal.SIGTERM),
|
||||
mock.call(951, signal.SIGTERM),
|
||||
mock.call(1641, signal.SIGTERM),
|
||||
mock.call(1700, signal.SIGTERM)]
|
||||
self.assertEqual(expected_mock_link_calls, mock_link.call_args_list)
|
||||
self.assertEqual(expected_mock_kill_calls, mock_kill.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'makedev', return_value='fake_dev')
|
||||
@mock.patch.object(os, 'mknod')
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch.object(utils, 'execute', return_value=('/dev/loop123\n', ''))
|
||||
def test_get_free_loop_device_ok(self, mock_exec, mock_path, mock_mknod,
|
||||
mock_mkdev):
|
||||
mock_path.exists.return_value = False
|
||||
self.assertEqual('/dev/loop123', bu.get_free_loop_device(1))
|
||||
mock_exec.assert_called_once_with('losetup', '--find')
|
||||
mock_path.exists.assert_called_once_with('/dev/loop0')
|
||||
mock_mknod.assert_called_once_with('/dev/loop0', 25008, 'fake_dev')
|
||||
mock_mkdev.assert_called_once_with(1, 0)
|
||||
|
||||
def test_set_apt_get_env(self):
|
||||
with mock.patch.dict('os.environ', {}):
|
||||
bu.set_apt_get_env()
|
||||
self.assertEqual('noninteractive', os.environ['DEBIAN_FRONTEND'])
|
||||
self.assertEqual('true', os.environ['DEBCONF_NONINTERACTIVE_SEEN'])
|
||||
for var in ('LC_ALL', 'LANG', 'LANGUAGE'):
|
||||
self.assertEqual('C', os.environ[var])
|
||||
|
||||
def test_strip_filename(self):
|
||||
self.assertEqual('safe_Tex.-98',
|
||||
bu.strip_filename('!@$^^^safe _Tex.?-98;'))
|
||||
|
||||
@mock.patch.object(os, 'makedev', return_value='fake_dev')
|
||||
@mock.patch.object(os, 'mknod')
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch.object(utils, 'execute', return_value=('', 'Error!!!'))
|
||||
def test_get_free_loop_device_not_found(self, mock_exec, mock_path,
|
||||
mock_mknod, mock_mkdev):
|
||||
mock_path.exists.return_value = False
|
||||
self.assertRaises(errors.NoFreeLoopDevices, bu.get_free_loop_device)
|
||||
|
||||
@mock.patch('tempfile.NamedTemporaryFile')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_create_sparse_tmp_file(self, mock_exec, mock_temp):
|
||||
tmp_file = mock.Mock()
|
||||
tmp_file.name = 'fake_name'
|
||||
mock_temp.return_value = tmp_file
|
||||
bu.create_sparse_tmp_file('dir', 'suffix', 1)
|
||||
mock_temp.assert_called_once_with(dir='dir', suffix='suffix',
|
||||
delete=False)
|
||||
mock_exec.assert_called_once_with('truncate', '-s', '1M',
|
||||
tmp_file.name)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_attach_file_to_loop(self, mock_exec):
|
||||
bu.attach_file_to_loop('file', 'loop')
|
||||
mock_exec.assert_called_once_with('losetup', 'loop', 'file')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_deattach_loop(self, mock_exec):
|
||||
mock_exec.return_value = ('/dev/loop0: [fd03]:130820 (/dev/loop0)', '')
|
||||
bu.deattach_loop('/dev/loop0', check_exit_code='Fake')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('losetup', '-a'),
|
||||
mock.call('losetup', '-d', '/dev/loop0', check_exit_code='Fake')
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, mock_exec_expected_calls)
|
||||
|
||||
@mock.patch.object(hu, 'parse_simple_kv')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_shrink_sparse_file(self, mock_exec, mock_parse):
|
||||
mock_parse.return_value = {'block count': 1, 'block size': 2}
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.shrink_sparse_file('file')
|
||||
mock_open.assert_called_once_with('file', 'rwb+')
|
||||
file_handle_mock.truncate.assert_called_once_with(1 * 2)
|
||||
expected_mock_exec_calls = [mock.call('e2fsck', '-y', '-f', 'file'),
|
||||
mock.call('resize2fs', '-F', '-M', 'file')]
|
||||
mock_parse.assert_called_once_with('dumpe2fs', 'file')
|
||||
self.assertEqual(expected_mock_exec_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_add_apt_source(self, mock_path):
|
||||
mock_path.return_value = 'fake_path'
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.add_apt_source('name1', 'uri1', 'suite1', 'section1', 'chroot')
|
||||
expected_calls = [mock.call('deb uri1 suite1 section1\n')]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.write.call_args_list)
|
||||
expected_mock_path_calls = [
|
||||
mock.call('chroot', 'etc/apt/sources.list.d',
|
||||
'fuel-image-name1.list')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_add_apt_source_no_section(self, mock_path):
|
||||
mock_path.return_value = 'fake_path'
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.add_apt_source('name2', 'uri2', 'suite2', None, 'chroot')
|
||||
expected_calls = [mock.call('deb uri2 suite2\n')]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.write.call_args_list)
|
||||
expected_mock_path_calls = [
|
||||
mock.call('chroot', 'etc/apt/sources.list.d',
|
||||
'fuel-image-name2.list')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch('fuel_agent.utils.build.utils.init_http_request',
|
||||
return_value=mock.Mock(text=_fake_ubuntu_release))
|
||||
def test_add_apt_preference(self, mock_get, mock_path):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
|
||||
fake_section = 'section1'
|
||||
bu.add_apt_preference(
|
||||
'name1',
|
||||
123,
|
||||
'test-archive',
|
||||
fake_section,
|
||||
'chroot',
|
||||
'http://test-uri'
|
||||
)
|
||||
|
||||
calls_args = [
|
||||
c[0][0] for c in file_handle_mock.write.call_args_list
|
||||
]
|
||||
|
||||
self.assertEqual(len(calls_args), 4)
|
||||
self.assertEqual(calls_args[0], 'Package: *\n')
|
||||
self.assertEqual(calls_args[1], 'Pin: release ')
|
||||
self.assertIn("l=TestLabel", calls_args[2])
|
||||
self.assertIn("n=testcodename", calls_args[2])
|
||||
self.assertIn("a=test-archive", calls_args[2])
|
||||
self.assertIn("o=TestOrigin", calls_args[2])
|
||||
self.assertIn("c=section1", calls_args[2])
|
||||
self.assertEqual(calls_args[3], 'Pin-Priority: 123\n')
|
||||
|
||||
expected_mock_path_calls = [
|
||||
mock.call('http://test-uri', 'dists', 'test-archive', 'Release'),
|
||||
mock.call('chroot', 'etc/apt/preferences.d',
|
||||
'fuel-image-name1.pref')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch('fuel_agent.utils.build.utils.init_http_request',
|
||||
return_value=mock.Mock(text=_fake_ubuntu_release))
|
||||
def test_add_apt_preference_multuple_sections(self, mock_get, mock_path):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
fake_sections = ['section2', 'section3']
|
||||
bu.add_apt_preference('name3', 234, 'test-archive',
|
||||
' '.join(fake_sections),
|
||||
'chroot', 'http://test-uri')
|
||||
|
||||
calls_args = [
|
||||
c[0][0] for c in file_handle_mock.write.call_args_list
|
||||
]
|
||||
|
||||
calls_package = [c for c in calls_args if c == 'Package: *\n']
|
||||
calls_pin = [c for c in calls_args if c == 'Pin: release ']
|
||||
calls_pin_p = [c for c in calls_args if c == 'Pin-Priority: 234\n']
|
||||
first_section = [
|
||||
c for c in calls_args if 'c={0}'.format(fake_sections[0]) in c
|
||||
]
|
||||
second_section = [
|
||||
c for c in calls_args if 'c={0}'.format(fake_sections[1]) in c
|
||||
]
|
||||
self.assertEqual(len(calls_package), len(fake_sections))
|
||||
self.assertEqual(len(calls_pin), len(fake_sections))
|
||||
self.assertEqual(len(calls_pin_p), len(fake_sections))
|
||||
self.assertEqual(len(first_section), 1)
|
||||
self.assertEqual(len(second_section), 1)
|
||||
|
||||
for pin_line in calls_args[2::4]:
|
||||
self.assertIn("l=TestLabel", pin_line)
|
||||
self.assertIn("n=testcodename", pin_line)
|
||||
self.assertIn("a=test-archive", pin_line)
|
||||
self.assertIn("o=TestOrigin", pin_line)
|
||||
|
||||
expected_mock_path_calls = [
|
||||
mock.call('http://test-uri', 'dists', 'test-archive', 'Release'),
|
||||
mock.call('chroot', 'etc/apt/preferences.d',
|
||||
'fuel-image-name3.pref')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch('fuel_agent.utils.build.utils.init_http_request',
|
||||
return_value=mock.Mock(text=_fake_ubuntu_release))
|
||||
def test_add_apt_preference_no_sections(self, mock_get, mock_path):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
|
||||
bu.add_apt_preference(
|
||||
'name1',
|
||||
123,
|
||||
'test-archive',
|
||||
'',
|
||||
'chroot',
|
||||
'http://test-uri'
|
||||
)
|
||||
|
||||
calls_args = [
|
||||
c[0][0] for c in file_handle_mock.write.call_args_list
|
||||
]
|
||||
|
||||
self.assertEqual(len(calls_args), 4)
|
||||
self.assertEqual(calls_args[0], 'Package: *\n')
|
||||
self.assertEqual(calls_args[1], 'Pin: release ')
|
||||
self.assertIn("l=TestLabel", calls_args[2])
|
||||
self.assertIn("n=testcodename", calls_args[2])
|
||||
self.assertIn("a=test-archive", calls_args[2])
|
||||
self.assertIn("o=TestOrigin", calls_args[2])
|
||||
self.assertNotIn("c=", calls_args[2])
|
||||
self.assertEqual(calls_args[3], 'Pin-Priority: 123\n')
|
||||
|
||||
expected_mock_path_calls = [
|
||||
mock.call('http://test-uri', 'test-archive', 'Release'),
|
||||
mock.call('chroot', 'etc/apt/preferences.d',
|
||||
'fuel-image-name1.pref')]
|
||||
self.assertEqual(expected_mock_path_calls,
|
||||
mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch.object(bu, 'clean_apt_settings')
|
||||
@mock.patch.object(os, 'path')
|
||||
def test_pre_apt_get(self, mock_path, mock_clean):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
bu.pre_apt_get('chroot')
|
||||
expected_calls = [
|
||||
mock.call('APT::Get::AllowUnauthenticated 1;\n'),
|
||||
mock.call('Acquire::ForceIPv4 "true";\n')]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.write.call_args_list)
|
||||
mock_clean.assert_called_once_with('chroot')
|
||||
expected_join_calls = [
|
||||
mock.call('chroot', 'etc/apt/apt.conf.d',
|
||||
CONF.allow_unsigned_file),
|
||||
mock.call('chroot', 'etc/apt/apt.conf.d',
|
||||
CONF.force_ipv4_file)]
|
||||
self.assertEqual(expected_join_calls, mock_path.join.call_args_list)
|
||||
|
||||
@mock.patch('gzip.open')
|
||||
@mock.patch.object(os, 'remove')
|
||||
def test_containerize_gzip(self, mock_remove, mock_gzip):
|
||||
with mock.patch('six.moves.builtins.open', create=True) as mock_open:
|
||||
file_handle_mock = mock_open.return_value.__enter__.return_value
|
||||
file_handle_mock.read.side_effect = ['test data', '']
|
||||
g = mock.Mock()
|
||||
mock_gzip.return_value = g
|
||||
self.assertEqual('file.gz', bu.containerize('file', 'gzip', 1))
|
||||
g.write.assert_called_once_with('test data')
|
||||
expected_calls = [mock.call(1), mock.call(1)]
|
||||
self.assertEqual(expected_calls,
|
||||
file_handle_mock.read.call_args_list)
|
||||
mock_remove.assert_called_once_with('file')
|
||||
|
||||
def test_containerize_bad_container(self):
|
||||
self.assertRaises(errors.WrongImageDataError, bu.containerize, 'file',
|
||||
'fake')
|
|
@ -1,58 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.objects import configdrive
|
||||
|
||||
|
||||
class TestConfigDriveScheme(test_base.BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestConfigDriveScheme, self).setUp()
|
||||
self.cd_scheme = configdrive.ConfigDriveScheme()
|
||||
|
||||
def test_template_names(self):
|
||||
self.cd_scheme._profile = 'pro_fi-le'
|
||||
actual = self.cd_scheme.template_names('what')
|
||||
expected = [
|
||||
'what_pro_fi-le.jinja2',
|
||||
'what_pro.jinja2',
|
||||
'what_pro_fi.jinja2',
|
||||
'what.jinja2'
|
||||
]
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
def test_template_data_no_common(self):
|
||||
self.assertRaises(errors.WrongConfigDriveDataError,
|
||||
self.cd_scheme.template_data)
|
||||
|
||||
def test_template_data_ok(self):
|
||||
cd_common = configdrive.ConfigDriveCommon(
|
||||
['auth_key1', 'auth_key2'], 'hostname', 'fqdn', 'name_servers',
|
||||
'search_domain', 'master_ip', 'master_url', 'udevrules',
|
||||
'admin_mac', 'admin_ip', 'admin_mask', 'admin_iface_name',
|
||||
'timezone', {'repo1': 'repo1_url', 'repo2': 'repo2_url'}, 'gw')
|
||||
cd_puppet = configdrive.ConfigDrivePuppet('master', 0)
|
||||
cd_mcollective = configdrive.ConfigDriveMcollective(
|
||||
'pskey', 'vhost', 'host', 'user', 'password', 'connector', 1)
|
||||
self.cd_scheme.common = cd_common
|
||||
self.cd_scheme.puppet = cd_puppet
|
||||
self.cd_scheme.mcollective = cd_mcollective
|
||||
template_data = self.cd_scheme.template_data()
|
||||
self.assertEqual(cd_common, template_data['common'])
|
||||
self.assertEqual(cd_puppet, template_data['puppet'])
|
||||
self.assertEqual(cd_mcollective, template_data['mcollective'])
|
|
@ -1,122 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import fs as fu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
class TestFSUtils(test_base.BaseTestCase):
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_make_fs(self, mock_exec):
|
||||
fu.make_fs('ext4', '-F', '-L fake_label', '/dev/fake')
|
||||
mock_exec.assert_called_once_with('mkfs.ext4', '-F', '-L',
|
||||
'fake_label', '/dev/fake')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_make_fs_swap(self, mock_exec):
|
||||
fu.make_fs('swap', '-f', '-L fake_label', '/dev/fake')
|
||||
mock_exec.assert_called_once_with('mkswap', '-f', '-L', 'fake_label',
|
||||
'/dev/fake')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_extend_fs_ok_ext2(self, mock_exec):
|
||||
fu.extend_fs('ext2', '/dev/fake')
|
||||
expected_calls = [
|
||||
mock.call('e2fsck', '-yf', '/dev/fake', check_exit_code=[0]),
|
||||
mock.call('resize2fs', '/dev/fake', check_exit_code=[0]),
|
||||
mock.call('e2fsck', '-pf', '/dev/fake', check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_extend_fs_ok_ext3(self, mock_exec):
|
||||
fu.extend_fs('ext3', '/dev/fake')
|
||||
expected_calls = [
|
||||
mock.call('e2fsck', '-yf', '/dev/fake', check_exit_code=[0]),
|
||||
mock.call('resize2fs', '/dev/fake', check_exit_code=[0]),
|
||||
mock.call('e2fsck', '-pf', '/dev/fake', check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_extend_fs_ok_ext4(self, mock_exec):
|
||||
fu.extend_fs('ext4', '/dev/fake')
|
||||
expected_calls = [
|
||||
mock.call('e2fsck', '-yf', '/dev/fake', check_exit_code=[0]),
|
||||
mock.call('resize2fs', '/dev/fake', check_exit_code=[0]),
|
||||
mock.call('e2fsck', '-pf', '/dev/fake', check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_extend_fs_ok_xfs(self, mock_exec):
|
||||
fu.extend_fs('xfs', '/dev/fake')
|
||||
mock_exec.assert_called_once_with(
|
||||
'xfs_growfs', '/dev/fake', check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_extend_fs_unsupported_fs(self, mock_exec):
|
||||
self.assertRaises(errors.FsUtilsError, fu.extend_fs,
|
||||
'unsupported', '/dev/fake')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_mount_fs(self, mock_exec):
|
||||
fu.mount_fs('ext3', '/dev/fake', '/target')
|
||||
mock_exec.assert_called_once_with(
|
||||
'mount', '-t', 'ext3', '/dev/fake', '/target', check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_mount_bind_no_path2(self, mock_exec):
|
||||
fu.mount_bind('/target', '/fake')
|
||||
mock_exec.assert_called_once_with(
|
||||
'mount', '--bind', '/fake', '/target/fake', check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_mount_bind_path2(self, mock_exec):
|
||||
fu.mount_bind('/target', '/fake', '/fake2')
|
||||
mock_exec.assert_called_once_with(
|
||||
'mount', '--bind', '/fake', '/target/fake2', check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_umount_fs_ok(self, mock_exec):
|
||||
fu.umount_fs('/fake')
|
||||
expected_calls = [
|
||||
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
|
||||
mock.call('umount', '/fake', check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_umount_fs_not_mounted(self, mock_exec):
|
||||
mock_exec.side_effect = errors.ProcessExecutionError
|
||||
fu.umount_fs('/fake')
|
||||
mock_exec.assert_called_once_with(
|
||||
'mountpoint', '-q', '/fake', check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_umount_fs_error(self, mock_exec):
|
||||
mock_exec.side_effect = [
|
||||
None, errors.ProcessExecutionError('message'), ('', '')]
|
||||
fu.umount_fs('/fake', try_lazy_umount=True)
|
||||
expected_calls = [
|
||||
mock.call('mountpoint', '-q', '/fake', check_exit_code=[0]),
|
||||
mock.call('umount', '/fake', check_exit_code=[0]),
|
||||
mock.call('umount', '-l', '/fake', check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(expected_calls, mock_exec.call_args_list)
|
|
@ -1,420 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
import os
|
||||
from oslotest import base as test_base
|
||||
import six
|
||||
import StringIO
|
||||
|
||||
if six.PY2:
|
||||
OPEN_FUNCTION_NAME = '__builtin__.open'
|
||||
else:
|
||||
OPEN_FUNCTION_NAME = 'builtins.open'
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import grub as gu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
class TestGrubUtils(test_base.BaseTestCase):
|
||||
|
||||
@mock.patch.object(os.path, 'isdir')
|
||||
def test_guess_grub2_conf(self, mock_isdir):
|
||||
side_effect_values = {
|
||||
'/target/boot/grub': True,
|
||||
'/target/boot/grub2': False
|
||||
}
|
||||
|
||||
def side_effect(key):
|
||||
return side_effect_values[key]
|
||||
|
||||
mock_isdir.side_effect = side_effect
|
||||
self.assertEqual(gu.guess_grub2_conf('/target'),
|
||||
'/boot/grub/grub.cfg')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/boot/grub': False,
|
||||
'/target/boot/grub2': True
|
||||
}
|
||||
self.assertEqual(gu.guess_grub2_conf('/target'),
|
||||
'/boot/grub2/grub.cfg')
|
||||
|
||||
@mock.patch.object(os.path, 'isfile')
|
||||
def test_guess_grub2_default(self, mock_isfile):
|
||||
side_effect_values = {
|
||||
'/target/etc/default/grub': True,
|
||||
'/target/etc/sysconfig/grub': False
|
||||
}
|
||||
|
||||
def side_effect(key):
|
||||
return side_effect_values[key]
|
||||
|
||||
mock_isfile.side_effect = side_effect
|
||||
self.assertEqual(gu.guess_grub2_default('/target'),
|
||||
'/etc/default/grub')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/etc/default/grub': False,
|
||||
'/target/etc/sysconfig/grub': True
|
||||
}
|
||||
self.assertEqual(gu.guess_grub2_default('/target'),
|
||||
'/etc/sysconfig/grub')
|
||||
|
||||
@mock.patch.object(os.path, 'isfile')
|
||||
def test_guess_grub2_mkconfig(self, mock_isfile):
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-mkconfig': True,
|
||||
'/target/sbin/grub2-mkconfig': False,
|
||||
'/target/usr/sbin/grub-mkconfig': False,
|
||||
'/target/usr/sbin/grub2-mkconfig': False
|
||||
}
|
||||
|
||||
def side_effect(key):
|
||||
return side_effect_values[key]
|
||||
|
||||
mock_isfile.side_effect = side_effect
|
||||
self.assertEqual(gu.guess_grub2_mkconfig('/target'),
|
||||
'/sbin/grub-mkconfig')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-mkconfig': False,
|
||||
'/target/sbin/grub2-mkconfig': True,
|
||||
'/target/usr/sbin/grub-mkconfig': False,
|
||||
'/target/usr/sbin/grub2-mkconfig': False
|
||||
}
|
||||
self.assertEqual(gu.guess_grub2_mkconfig('/target'),
|
||||
'/sbin/grub2-mkconfig')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-mkconfig': False,
|
||||
'/target/sbin/grub2-mkconfig': False,
|
||||
'/target/usr/sbin/grub-mkconfig': True,
|
||||
'/target/usr/sbin/grub2-mkconfig': False
|
||||
}
|
||||
self.assertEqual(gu.guess_grub2_mkconfig('/target'),
|
||||
'/usr/sbin/grub-mkconfig')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-mkconfig': False,
|
||||
'/target/sbin/grub2-mkconfig': False,
|
||||
'/target/usr/sbin/grub-mkconfig': False,
|
||||
'/target/usr/sbin/grub2-mkconfig': True
|
||||
}
|
||||
self.assertEqual(gu.guess_grub2_mkconfig('/target'),
|
||||
'/usr/sbin/grub2-mkconfig')
|
||||
|
||||
@mock.patch.object(gu, 'guess_grub_install')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_guess_grub_version_1(self, mock_exec, mock_ggi):
|
||||
mock_ggi.return_value = '/grub_install'
|
||||
mock_exec.return_value = ('foo 0.97 bar', '')
|
||||
version = gu.guess_grub_version('/target')
|
||||
cmd = 'chroot /target /grub_install --version'.split()
|
||||
mock_exec.assert_called_once_with(*cmd)
|
||||
self.assertEqual(version, 1)
|
||||
|
||||
@mock.patch.object(gu, 'guess_grub_install')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_guess_grub_version_2(self, mock_exec, mock_ggi):
|
||||
mock_ggi.return_value = '/grub_install'
|
||||
mock_exec.return_value = ('foo bar', '')
|
||||
version = gu.guess_grub_version('/target')
|
||||
cmd = 'chroot /target /grub_install --version'.split()
|
||||
mock_exec.assert_called_once_with(*cmd)
|
||||
self.assertEqual(version, 2)
|
||||
|
||||
@mock.patch.object(os.path, 'isfile')
|
||||
def test_guess_grub(self, mock_isfile):
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub': True,
|
||||
'/target/usr/sbin/grub': False
|
||||
}
|
||||
|
||||
def side_effect(key):
|
||||
return side_effect_values[key]
|
||||
|
||||
mock_isfile.side_effect = side_effect
|
||||
self.assertEqual(gu.guess_grub('/target'),
|
||||
'/sbin/grub')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub': False,
|
||||
'/target/usr/sbin/grub': True
|
||||
}
|
||||
self.assertEqual(gu.guess_grub('/target'),
|
||||
'/usr/sbin/grub')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub': False,
|
||||
'/target/usr/sbin/grub': False
|
||||
}
|
||||
self.assertRaises(errors.GrubUtilsError, gu.guess_grub, '/target')
|
||||
|
||||
@mock.patch.object(os.path, 'isfile')
|
||||
def test_grub_install(self, mock_isfile):
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-install': True,
|
||||
'/target/sbin/grub2-install': False,
|
||||
'/target/usr/sbin/grub-install': False,
|
||||
'/target/usr/sbin/grub2-install': False
|
||||
}
|
||||
|
||||
def side_effect(key):
|
||||
return side_effect_values[key]
|
||||
|
||||
mock_isfile.side_effect = side_effect
|
||||
self.assertEqual(gu.guess_grub_install('/target'),
|
||||
'/sbin/grub-install')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-install': False,
|
||||
'/target/sbin/grub2-install': True,
|
||||
'/target/usr/sbin/grub-install': False,
|
||||
'/target/usr/sbin/grub2-install': False
|
||||
}
|
||||
self.assertEqual(gu.guess_grub_install('/target'),
|
||||
'/sbin/grub2-install')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-install': False,
|
||||
'/target/sbin/grub2-install': False,
|
||||
'/target/usr/sbin/grub-install': True,
|
||||
'/target/usr/sbin/grub2-install': False
|
||||
}
|
||||
self.assertEqual(gu.guess_grub_install('/target'),
|
||||
'/usr/sbin/grub-install')
|
||||
|
||||
side_effect_values = {
|
||||
'/target/sbin/grub-install': False,
|
||||
'/target/sbin/grub2-install': False,
|
||||
'/target/usr/sbin/grub-install': False,
|
||||
'/target/usr/sbin/grub2-install': True
|
||||
}
|
||||
self.assertEqual(gu.guess_grub_install('/target'),
|
||||
'/usr/sbin/grub2-install')
|
||||
|
||||
@mock.patch('fuel_agent.utils.grub.utils.guess_filename')
|
||||
def test_guess_kernel(self, mock_guess):
|
||||
mock_guess.return_value = 'vmlinuz-version'
|
||||
self.assertEqual(gu.guess_kernel('/target'), 'vmlinuz-version')
|
||||
mock_guess.assert_called_once_with(
|
||||
path='/target/boot', regexp=r'^vmlinuz.*')
|
||||
mock_guess.reset_mock()
|
||||
|
||||
mock_guess.return_value = 'vmlinuz-version'
|
||||
self.assertEqual(gu.guess_kernel('/target', r'^vmlinuz-version.*'),
|
||||
'vmlinuz-version')
|
||||
mock_guess.assert_called_once_with(
|
||||
path='/target/boot', regexp=r'^vmlinuz-version.*')
|
||||
mock_guess.reset_mock()
|
||||
|
||||
mock_guess.return_value = None
|
||||
self.assertRaises(errors.GrubUtilsError, gu.guess_kernel, '/target')
|
||||
|
||||
@mock.patch('fuel_agent.utils.grub.utils.guess_filename')
|
||||
def test_guess_initrd(self, mock_guess):
|
||||
mock_guess.return_value = 'initrd-version'
|
||||
self.assertEqual(gu.guess_initrd('/target'), 'initrd-version')
|
||||
mock_guess.assert_called_once_with(
|
||||
path='/target/boot', regexp=r'^(initrd|initramfs).*')
|
||||
mock_guess.reset_mock()
|
||||
|
||||
mock_guess.return_value = 'initramfs-version'
|
||||
self.assertEqual(gu.guess_initrd('/target', r'^initramfs-version.*'),
|
||||
'initramfs-version')
|
||||
mock_guess.assert_called_once_with(
|
||||
path='/target/boot', regexp=r'^initramfs-version.*')
|
||||
mock_guess.reset_mock()
|
||||
|
||||
mock_guess.return_value = None
|
||||
self.assertRaises(errors.GrubUtilsError, gu.guess_initrd, '/target')
|
||||
|
||||
@mock.patch.object(gu, 'grub1_stage1')
|
||||
@mock.patch.object(gu, 'grub1_mbr')
|
||||
def test_grub1_install(self, mock_mbr, mock_stage1):
|
||||
install_devices = ['/dev/foo', '/dev/bar']
|
||||
expected_calls_mbr = []
|
||||
for install_device in install_devices:
|
||||
expected_calls_mbr.append(
|
||||
mock.call(install_device, '/dev/foo', '0', chroot='/target'))
|
||||
gu.grub1_install(install_devices, '/dev/foo1', '/target')
|
||||
self.assertEqual(expected_calls_mbr, mock_mbr.call_args_list)
|
||||
mock_stage1.assert_called_once_with(chroot='/target')
|
||||
# should raise exception if boot_device (second argument)
|
||||
# is not a partition but a whole disk
|
||||
self.assertRaises(errors.GrubUtilsError, gu.grub1_install,
|
||||
'/dev/foo', '/dev/foo', chroot='/target')
|
||||
|
||||
@mock.patch.object(gu, 'guess_grub')
|
||||
@mock.patch.object(os, 'chmod')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_grub1_mbr_install_differs_boot(self, mock_exec,
|
||||
mock_chmod, mock_guess):
|
||||
mock_guess.return_value = '/sbin/grub'
|
||||
mock_exec.return_value = ('stdout', 'stderr')
|
||||
|
||||
# install_device != boot_disk
|
||||
batch = 'device (hd0) /dev/foo\n'
|
||||
batch += 'geometry (hd0) 130 255 63\n'
|
||||
batch += 'device (hd1) /dev/bar\n'
|
||||
batch += 'geometry (hd1) 130 255 63\n'
|
||||
batch += 'root (hd1,0)\n'
|
||||
batch += 'setup (hd0)\n'
|
||||
batch += 'quit\n'
|
||||
script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch'
|
||||
|
||||
mock_open = mock.mock_open()
|
||||
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
|
||||
gu.grub1_mbr('/dev/foo', '/dev/bar', '0', chroot='/target')
|
||||
self.assertEqual(
|
||||
mock_open.call_args_list,
|
||||
[mock.call('/target/tmp/grub.batch', 'wb'),
|
||||
mock.call('/target/tmp/grub.sh', 'wb')]
|
||||
)
|
||||
mock_open_file = mock_open()
|
||||
self.assertEqual(
|
||||
mock_open_file.write.call_args_list,
|
||||
[mock.call(batch), mock.call(script)]
|
||||
)
|
||||
mock_chmod.assert_called_once_with('/target/tmp/grub.sh', 0o755)
|
||||
mock_exec.assert_called_once_with(
|
||||
'chroot', '/target', '/tmp/grub.sh',
|
||||
run_as_root=True, check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(gu, 'guess_grub')
|
||||
@mock.patch.object(os, 'chmod')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_grub1_mbr_install_same_as_boot(self, mock_exec,
|
||||
mock_chmod, mock_guess):
|
||||
mock_guess.return_value = '/sbin/grub'
|
||||
mock_exec.return_value = ('stdout', 'stderr')
|
||||
|
||||
# install_device == boot_disk
|
||||
batch = 'device (hd0) /dev/foo\n'
|
||||
batch += 'geometry (hd0) 130 255 63\n'
|
||||
batch += 'root (hd0,0)\n'
|
||||
batch += 'setup (hd0)\n'
|
||||
batch += 'quit\n'
|
||||
script = 'cat /tmp/grub.batch | /sbin/grub --no-floppy --batch'
|
||||
|
||||
mock_open = mock.mock_open()
|
||||
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
|
||||
gu.grub1_mbr('/dev/foo', '/dev/foo', '0', chroot='/target')
|
||||
self.assertEqual(
|
||||
mock_open.call_args_list,
|
||||
[mock.call('/target/tmp/grub.batch', 'wb'),
|
||||
mock.call('/target/tmp/grub.sh', 'wb')]
|
||||
)
|
||||
mock_open_file = mock_open()
|
||||
self.assertEqual(
|
||||
mock_open_file.write.call_args_list,
|
||||
[mock.call(batch), mock.call(script)]
|
||||
)
|
||||
mock_chmod.assert_called_once_with('/target/tmp/grub.sh', 0o755)
|
||||
mock_exec.assert_called_once_with(
|
||||
'chroot', '/target', '/tmp/grub.sh',
|
||||
run_as_root=True, check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(gu, 'guess_kernel')
|
||||
@mock.patch.object(gu, 'guess_initrd')
|
||||
def test_grub1_cfg_kernel_initrd_are_not_set(self, mock_initrd,
|
||||
mock_kernel):
|
||||
mock_kernel.return_value = 'kernel-version'
|
||||
mock_initrd.return_value = 'initrd-version'
|
||||
config = """
|
||||
default=0
|
||||
timeout=5
|
||||
title Default (kernel-version)
|
||||
kernel /kernel-version kernel-params
|
||||
initrd /initrd-version
|
||||
"""
|
||||
|
||||
mock_open = mock.mock_open()
|
||||
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
|
||||
gu.grub1_cfg(chroot='/target', kernel_params='kernel-params')
|
||||
mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb')
|
||||
mock_open_file = mock_open()
|
||||
mock_open_file.write.assert_called_once_with(config)
|
||||
|
||||
def test_grub1_cfg_kernel_initrd_are_set(self):
|
||||
config = """
|
||||
default=0
|
||||
timeout=10
|
||||
title Default (kernel-version-set)
|
||||
kernel /kernel-version-set kernel-params
|
||||
initrd /initrd-version-set
|
||||
"""
|
||||
|
||||
mock_open = mock.mock_open()
|
||||
with mock.patch(OPEN_FUNCTION_NAME, new=mock_open, create=True):
|
||||
gu.grub1_cfg(kernel='kernel-version-set',
|
||||
initrd='initrd-version-set',
|
||||
chroot='/target', kernel_params='kernel-params',
|
||||
grub_timeout=10)
|
||||
mock_open.assert_called_once_with('/target/boot/grub/grub.conf', 'wb')
|
||||
mock_open_file = mock_open()
|
||||
mock_open_file.write.assert_called_once_with(config)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(gu, 'guess_grub_install')
|
||||
def test_grub2_install(self, mock_guess_grub, mock_exec):
|
||||
mock_guess_grub.return_value = '/sbin/grub'
|
||||
expected_calls = [
|
||||
mock.call('chroot', '/target', '/sbin/grub', '/dev/foo',
|
||||
run_as_root=True, check_exit_code=[0]),
|
||||
mock.call('chroot', '/target', '/sbin/grub', '/dev/bar',
|
||||
run_as_root=True, check_exit_code=[0])
|
||||
]
|
||||
gu.grub2_install(['/dev/foo', '/dev/bar'], chroot='/target')
|
||||
self.assertEqual(mock_exec.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(gu, 'guess_grub2_conf')
|
||||
@mock.patch.object(gu, 'guess_grub2_mkconfig')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(gu, 'guess_grub2_default')
|
||||
def test_grub2_cfg(self, mock_def, mock_exec, mock_mkconfig, mock_conf):
|
||||
mock_def.return_value = '/etc/default/grub'
|
||||
mock_mkconfig.return_value = '/sbin/grub-mkconfig'
|
||||
mock_conf.return_value = '/boot/grub/grub.cfg'
|
||||
orig_content = """foo
|
||||
GRUB_CMDLINE_LINUX="kernel-params-orig"
|
||||
bar"""
|
||||
new_content = """foo
|
||||
GRUB_CMDLINE_LINUX="kernel-params-new"
|
||||
bar
|
||||
GRUB_RECORDFAIL_TIMEOUT=10
|
||||
"""
|
||||
|
||||
# mock_open = mock.mock_open(read_data=orig_content)
|
||||
with mock.patch(OPEN_FUNCTION_NAME,
|
||||
new=mock.mock_open(read_data=orig_content),
|
||||
create=True) as mock_open:
|
||||
mock_open.return_value = mock.MagicMock(spec=file)
|
||||
handle = mock_open.return_value.__enter__.return_value
|
||||
handle.__iter__.return_value = StringIO.StringIO(orig_content)
|
||||
gu.grub2_cfg(kernel_params='kernel-params-new', chroot='/target',
|
||||
grub_timeout=10)
|
||||
|
||||
self.assertEqual(
|
||||
mock_open.call_args_list,
|
||||
[mock.call('/target/etc/default/grub'),
|
||||
mock.call('/target/etc/default/grub', 'wb')]
|
||||
)
|
||||
|
||||
handle.write.assert_called_once_with(new_content)
|
||||
mock_exec.assert_called_once_with('chroot', '/target',
|
||||
'/sbin/grub-mkconfig',
|
||||
'-o', '/boot/grub/grub.cfg',
|
||||
run_as_root=True)
|
|
@ -1,474 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent.utils import hardware as hu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
class TestHardwareUtils(test_base.BaseTestCase):
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_parse_dmidecode(self, exec_mock):
|
||||
exec_mock.return_value = ["""
|
||||
System Slot Information
|
||||
Designation: PCIEX16_1
|
||||
ID: 1
|
||||
Bus Address: 0000:00:01.0
|
||||
Characteristics:
|
||||
3.3 V is provided
|
||||
PME signal is supported
|
||||
|
||||
System Slot Information
|
||||
Type: 32-bit PCI Express
|
||||
ID: 3
|
||||
Characteristics:
|
||||
Opening is shared
|
||||
Bus Address: 0000:00:1c.4
|
||||
|
||||
|
||||
"""]
|
||||
|
||||
expected = [{"designation": "PCIEX16_1",
|
||||
"id": "1",
|
||||
"characteristics": ["3.3 V is provided",
|
||||
"PME signal is supported"],
|
||||
"bus address": "0000:00:01.0"},
|
||||
{"type": "32-bit PCI Express",
|
||||
"id": "3",
|
||||
"characteristics": ["Opening is shared"],
|
||||
"bus address": "0000:00:1c.4"}]
|
||||
|
||||
self.assertEqual(expected, hu.parse_dmidecode("fake_type"))
|
||||
exec_mock.assert_called_once_with("dmidecode", "-q", "--type",
|
||||
"fake_type")
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_parse_lspci(self, exec_mock):
|
||||
exec_mock.return_value = ["""Slot: 07:00.0
|
||||
Class: PCI bridge
|
||||
Vendor: ASMedia Technology Inc.
|
||||
Device: ASM1083/1085 PCIe to PCI Bridge
|
||||
Rev: 01
|
||||
ProgIf: 01
|
||||
|
||||
Slot: 09:00.0
|
||||
Class: IDE interface
|
||||
Vendor: Marvell Technology Group Ltd.
|
||||
Device: 88SE6121 SATA II / PATA Controller
|
||||
SVendor: ASUSTeK Computer Inc.
|
||||
SDevice: Device 82a2
|
||||
Rev: b2
|
||||
ProgIf: 8f
|
||||
|
||||
"""]
|
||||
|
||||
expected = [{'class': 'PCI bridge',
|
||||
'device': 'ASM1083/1085 PCIe to PCI Bridge',
|
||||
'progif': '01',
|
||||
'rev': '01',
|
||||
'slot': '07:00.0',
|
||||
'vendor': 'ASMedia Technology Inc.'},
|
||||
{'class': 'IDE interface',
|
||||
'device': '88SE6121 SATA II / PATA Controller',
|
||||
'progif': '8f',
|
||||
'rev': 'b2',
|
||||
'sdevice': 'Device 82a2',
|
||||
'slot': '09:00.0',
|
||||
'svendor': 'ASUSTeK Computer Inc.',
|
||||
'vendor': 'Marvell Technology Group Ltd.'}]
|
||||
|
||||
self.assertEqual(expected, hu.parse_lspci())
|
||||
exec_mock.assert_called_once_with('lspci', '-vmm', '-D')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_parse_simple_kv(self, exec_mock):
|
||||
exec_mock.return_value = ["""driver: r8169
|
||||
version: 2.3LK-NAPI
|
||||
firmware-version: rtl_nic/rtl8168e-2.fw
|
||||
bus-info: 0000:06:00.0
|
||||
supports-statistics: yes
|
||||
supports-test: no
|
||||
supports-eeprom-access: no
|
||||
supports-register-dump: yes
|
||||
|
||||
"""]
|
||||
|
||||
expected = {'driver': 'r8169',
|
||||
'version': '2.3LK-NAPI',
|
||||
'firmware-version': 'rtl_nic/rtl8168e-2.fw',
|
||||
'bus-info': '0000:06:00.0',
|
||||
'supports-statistics': 'yes',
|
||||
'supports-test': 'no',
|
||||
'supports-eeprom-access': 'no',
|
||||
'supports-register-dump': 'yes'}
|
||||
|
||||
self.assertEqual(expected, hu.parse_simple_kv('fake', 'cmd'))
|
||||
exec_mock.assert_called_once_with('fake', 'cmd')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_udevreport(self, mock_exec):
|
||||
# should run udevadm info OS command
|
||||
# in order to get udev properties for a device
|
||||
mock_exec.return_value = (
|
||||
'DEVLINKS=\'/dev/disk/by-id/fakeid1 /dev/disk/by-id/fakeid2\'\n'
|
||||
'DEVNAME=\'/dev/fake\'\n'
|
||||
'DEVPATH=\'/devices/fakepath\'\n'
|
||||
'DEVTYPE=\'disk\'\n'
|
||||
'MAJOR=\'11\'\n'
|
||||
'MINOR=\'0\'\n'
|
||||
'ID_BUS=\'fakebus\'\n'
|
||||
'ID_MODEL=\'fakemodel\'\n'
|
||||
'ID_SERIAL_SHORT=\'fakeserial\'\n'
|
||||
'ID_WWN=\'fakewwn\'\n'
|
||||
'ID_CDROM=\'1\'\n'
|
||||
'ANOTHER=\'another\'\n',
|
||||
''
|
||||
)
|
||||
expected = {
|
||||
'DEVLINKS': ['/dev/disk/by-id/fakeid1', '/dev/disk/by-id/fakeid2'],
|
||||
'DEVNAME': '/dev/fake',
|
||||
'DEVPATH': '/devices/fakepath',
|
||||
'DEVTYPE': 'disk',
|
||||
'MAJOR': '11',
|
||||
'MINOR': '0',
|
||||
'ID_BUS': 'fakebus',
|
||||
'ID_MODEL': 'fakemodel',
|
||||
'ID_SERIAL_SHORT': 'fakeserial',
|
||||
'ID_WWN': 'fakewwn',
|
||||
'ID_CDROM': '1'
|
||||
}
|
||||
self.assertEqual(expected, hu.udevreport('/dev/fake'))
|
||||
mock_exec.assert_called_once_with('udevadm',
|
||||
'info',
|
||||
'--query=property',
|
||||
'--export',
|
||||
'--name=/dev/fake',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_blockdevreport(self, mock_exec):
|
||||
# should run blockdev OS command
|
||||
# in order to get block device properties
|
||||
cmd = ['blockdev', '--getsz', '--getro', '--getss', '--getpbsz',
|
||||
'--getsize64', '--getiomin', '--getioopt', '--getra',
|
||||
'--getalignoff', '--getmaxsect', '/dev/fake']
|
||||
mock_exec.return_value = (
|
||||
'625142448\n0\n512\n4096\n320072933376\n4096\n0\n256\n0\n1024',
|
||||
''
|
||||
)
|
||||
expected = {
|
||||
'sz': '625142448',
|
||||
'ro': '0',
|
||||
'ss': '512',
|
||||
'pbsz': '4096',
|
||||
'size64': '320072933376',
|
||||
'iomin': '4096',
|
||||
'ioopt': '0',
|
||||
'ra': '256',
|
||||
'alignoff': '0',
|
||||
'maxsect': '1024'
|
||||
}
|
||||
self.assertEqual(expected, hu.blockdevreport('/dev/fake'))
|
||||
mock_exec.assert_called_once_with(*cmd, check_exit_code=[0])
|
||||
|
||||
@mock.patch('six.moves.builtins.open')
|
||||
def test_extrareport(self, mock_open):
|
||||
# should read some files from sysfs e.g. /sys/block/fake/removable
|
||||
# in order to get some device properties
|
||||
def with_side_effect(arg):
|
||||
mock_with = mock.MagicMock()
|
||||
mock_with.__exit__.return_value = None
|
||||
mock_file = mock.Mock()
|
||||
if arg == '/sys/block/fake/removable':
|
||||
mock_file.read.return_value = '0\n'
|
||||
elif arg == '/sys/block/fake/device/state':
|
||||
mock_file.read.return_value = 'running\n'
|
||||
elif arg == '/sys/block/fake/device/timeout':
|
||||
mock_file.read.return_value = '30\n'
|
||||
mock_with.__enter__.return_value = mock_file
|
||||
return mock_with
|
||||
mock_open.side_effect = with_side_effect
|
||||
expected = {'removable': '0', 'state': 'running', 'timeout': '30'}
|
||||
self.assertEqual(expected, hu.extrareport('/dev/fake'))
|
||||
|
||||
@mock.patch('six.moves.builtins.open')
|
||||
def test_extrareport_exceptions(self, mock_open):
|
||||
mock_open.side_effect = Exception('foo')
|
||||
expected = {}
|
||||
self.assertEqual(expected, hu.extrareport('/dev/fake'))
|
||||
|
||||
@mock.patch.object(hu, 'blockdevreport')
|
||||
@mock.patch.object(hu, 'udevreport')
|
||||
def test_is_disk_uspec_bspec_none(self, mock_ureport, mock_breport):
|
||||
# should call udevreport if uspec is None
|
||||
# should call blockdevreport if bspec is None
|
||||
# should return True if uspec and bspec are empty
|
||||
mock_ureport.return_value = {}
|
||||
mock_breport.return_value = {}
|
||||
self.assertTrue(hu.is_disk('/dev/fake'))
|
||||
mock_ureport.assert_called_once_with('/dev/fake')
|
||||
mock_breport.assert_called_once_with('/dev/fake')
|
||||
|
||||
@mock.patch.object(hu, 'udevreport')
|
||||
def test_is_disk_uspec_none(self, mock_ureport):
|
||||
# should call udevreport if uspec is None but bspec is not None
|
||||
bspec = {'key': 'value'}
|
||||
mock_ureport.return_value = {}
|
||||
hu.is_disk('/dev/fake', bspec=bspec)
|
||||
mock_ureport.assert_called_once_with('/dev/fake')
|
||||
|
||||
@mock.patch.object(hu, 'blockdevreport')
|
||||
def test_is_disk_bspec_none(self, mock_breport):
|
||||
# should call blockdevreport if bspec is None but uspec is not None
|
||||
uspec = {'key': 'value'}
|
||||
mock_breport.return_value = {}
|
||||
hu.is_disk('/dev/fake', uspec=uspec)
|
||||
mock_breport.assert_called_once_with('/dev/fake')
|
||||
|
||||
@mock.patch.object(hu, 'blockdevreport')
|
||||
def test_is_disk_cdrom(self, mock_breport):
|
||||
# should return False if udev ID_CDROM is set to 1
|
||||
mock_breport.return_value = {}
|
||||
uspec = {
|
||||
'ID_CDROM': '1'
|
||||
}
|
||||
self.assertFalse(hu.is_disk('/dev/fake', uspec=uspec))
|
||||
|
||||
@mock.patch.object(hu, 'blockdevreport')
|
||||
def test_is_disk_partition(self, mock_breport):
|
||||
# should return False if udev DEVTYPE is partition
|
||||
mock_breport.return_value = {}
|
||||
uspec = {
|
||||
'DEVTYPE': 'partition'
|
||||
}
|
||||
self.assertFalse(hu.is_disk('/dev/fake', uspec=uspec))
|
||||
|
||||
@mock.patch.object(hu, 'blockdevreport')
|
||||
def test_is_disk_major(self, mock_breport):
|
||||
# should return False if udev MAJOR is not in a list of
|
||||
# major numbers which are used for disks
|
||||
# look at kernel/Documentation/devices.txt
|
||||
mock_breport.return_value = {}
|
||||
valid_majors = [3, 8, 65, 66, 67, 68, 69, 70, 71, 104, 105,
|
||||
106, 107, 108, 109, 110, 111, 202, 252, 253, 259]
|
||||
for major in (set(range(1, 261)) - set(valid_majors)):
|
||||
uspec = {
|
||||
'MAJOR': str(major)
|
||||
}
|
||||
self.assertFalse(hu.is_disk('/dev/fake', uspec=uspec))
|
||||
|
||||
@mock.patch.object(hu, 'udevreport')
|
||||
def test_is_disk_readonly(self, mock_ureport):
|
||||
# should return False if device is read only
|
||||
mock_ureport.return_value = {}
|
||||
bspec = {
|
||||
'ro': '1'
|
||||
}
|
||||
self.assertFalse(hu.is_disk('/dev/fake', bspec=bspec))
|
||||
|
||||
@mock.patch('fuel_agent.utils.hardware.utils.execute')
|
||||
def test_get_block_devices_from_udev_db(self, mock_exec):
|
||||
mock_exec.return_value = ("""P: /devices/virtual/block/loop0
|
||||
N: loop0
|
||||
E: DEVNAME=/dev/loop0
|
||||
E: DEVPATH=/devices/virtual/block/loop0
|
||||
E: DEVTYPE=disk
|
||||
E: MAJOR=7
|
||||
E: SUBSYSTEM=block
|
||||
|
||||
P: /devices/pci0000:00/0000:00:1f.2/ata1/host0/target0:0:0/0:0:0:0/block/sda
|
||||
N: sda
|
||||
S: disk/by-id/wwn-0x5000c5004008ac0f
|
||||
S: disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0
|
||||
E: DEVNAME=/dev/sda
|
||||
E: DEVTYPE=disk
|
||||
E: ID_ATA=1
|
||||
E: MAJOR=8
|
||||
E: SUBSYSTEM=block
|
||||
E: UDEV_LOG=3
|
||||
|
||||
P: /devices/pci:00/:00:04.0/misc/nvme0
|
||||
N: nvme0
|
||||
E: DEVNAME=/dev/nvme0
|
||||
E: DEVPATH=/devices/pci:00/:00:04.0/misc/nvme0
|
||||
E: MAJOR=10
|
||||
E: MINOR=57
|
||||
E: SUBSYSTEM=misc
|
||||
|
||||
P: /devices/pci:00/:00:04.0/block/nvme0n1
|
||||
N: nvme0n1
|
||||
E: DEVNAME=/dev/nvme0n1
|
||||
E: DEVPATH=/devices/pci:00/:00:04.0/block/nvme0n1
|
||||
E: DEVTYPE=disk
|
||||
E: MAJOR=259
|
||||
E: MINOR=0
|
||||
E: SUBSYSTEM=block
|
||||
E: USEC_INITIALIZED=87744
|
||||
|
||||
P: /devices/pci0000:00/0000:00:1c.1/target16:0:0/16:0:0:0/block/sr0
|
||||
E: DEVTYPE=disk
|
||||
E: DEVNAME=/dev/sr0
|
||||
E: MAJOR=11
|
||||
E: MINOR=0
|
||||
E: SEQNUM=4400
|
||||
E: SUBSYSTEM=block
|
||||
|
||||
P: /devices/pci0000:00/0000:00:1f.2/ata1/host0/target0:0:0/0:0:0:0/block/sda
|
||||
N: sda
|
||||
S: disk/by-id/wwn-0x5000c5004008ac0f
|
||||
S: disk/by-path/pci-0000:00:1f.2-scsi-0:0:0:0
|
||||
E: DEVNAME=/dev/sda1
|
||||
E: DEVTYPE=partition
|
||||
E: ID_ATA=1
|
||||
E: SUBSYSTEM=block
|
||||
E: MAJOR=8
|
||||
E: UDEV_LOG=3""", '')
|
||||
|
||||
self.assertEqual(['/dev/sda', '/dev/nvme0n1', '/dev/sda1'],
|
||||
hu.get_block_devices_from_udev_db())
|
||||
|
||||
@mock.patch.object(hu, 'get_block_devices_from_udev_db')
|
||||
@mock.patch.object(hu, 'is_disk')
|
||||
@mock.patch.object(hu, 'extrareport')
|
||||
@mock.patch.object(hu, 'blockdevreport')
|
||||
@mock.patch.object(hu, 'udevreport')
|
||||
def test_list_block_devices(self, mock_ureport, mock_breport, mock_ereport,
|
||||
mock_isdisk, mock_get_devs):
|
||||
# should run blockdev --report command
|
||||
# in order to get a list of block devices
|
||||
# should call report methods to get device info
|
||||
# should call is_disk method to filter out
|
||||
# those block devices which are not disks
|
||||
mock_get_devs.return_value = ['/dev/fake', '/dev/fake1', '/dev/sr0']
|
||||
|
||||
def isdisk_side_effect(arg, uspec=None, bspec=None):
|
||||
if arg == '/dev/fake':
|
||||
return True
|
||||
elif arg in ('/dev/fake1', '/dev/sr0'):
|
||||
return False
|
||||
mock_isdisk.side_effect = isdisk_side_effect
|
||||
mock_ureport.return_value = {'key0': 'value0'}
|
||||
mock_breport.return_value = {'key1': 'value1'}
|
||||
mock_ereport.return_value = {'key2': 'value2'}
|
||||
|
||||
expected = [{
|
||||
'device': '/dev/fake',
|
||||
'uspec': {'key0': 'value0'},
|
||||
'bspec': {'key1': 'value1'},
|
||||
'espec': {'key2': 'value2'}
|
||||
}]
|
||||
self.assertEqual(hu.list_block_devices(), expected)
|
||||
self.assertEqual(mock_ureport.call_args_list, [mock.call('/dev/fake'),
|
||||
mock.call('/dev/fake1'), mock.call('/dev/sr0')])
|
||||
self.assertEqual(mock_breport.call_args_list, [mock.call('/dev/fake'),
|
||||
mock.call('/dev/fake1'), mock.call('/dev/sr0')])
|
||||
self.assertEqual(mock_ereport.call_args_list, [mock.call('/dev/fake'),
|
||||
mock.call('/dev/fake1'), mock.call('/dev/sr0')])
|
||||
|
||||
@mock.patch.object(hu, 'get_block_devices_from_udev_db')
|
||||
@mock.patch.object(hu, 'is_disk')
|
||||
@mock.patch.object(hu, 'extrareport')
|
||||
@mock.patch.object(hu, 'blockdevreport')
|
||||
@mock.patch.object(hu, 'udevreport')
|
||||
def test_list_block_devices_removable_vendors(self, mock_ureport,
|
||||
mock_breport, mock_ereport,
|
||||
mock_isdisk, mock_get_devs):
|
||||
mock_get_devs.return_value = ['/dev/no_vendor_id',
|
||||
'/dev/wrong_vendor_id',
|
||||
'/dev/right_vendor_id']
|
||||
mock_isdisk.return_value = True
|
||||
mock_ureport.side_effect = [
|
||||
{},
|
||||
{'ID_VENDOR': 'Cisco'},
|
||||
{'ID_VENDOR': 'IBM'},
|
||||
]
|
||||
mock_ereport.return_value = {'removable': '1'}
|
||||
mock_breport.return_value = {'key1': 'value1'}
|
||||
expected = [{
|
||||
'device': '/dev/right_vendor_id',
|
||||
'uspec': {'ID_VENDOR': 'IBM'},
|
||||
'bspec': {'key1': 'value1'},
|
||||
'espec': {'removable': '1'}
|
||||
}]
|
||||
self.assertEqual(hu.list_block_devices(), expected)
|
||||
self.assertEqual(
|
||||
mock_ureport.call_args_list,
|
||||
[mock.call('/dev/no_vendor_id'),
|
||||
mock.call('/dev/wrong_vendor_id'),
|
||||
mock.call('/dev/right_vendor_id')])
|
||||
mock_breport.assert_called_once_with('/dev/right_vendor_id')
|
||||
self.assertEqual(
|
||||
mock_ereport.call_args_list,
|
||||
[mock.call('/dev/no_vendor_id'),
|
||||
mock.call('/dev/wrong_vendor_id'),
|
||||
mock.call('/dev/right_vendor_id')])
|
||||
|
||||
def test_match_device_devlinks(self):
|
||||
# should return true if at least one by-id link from first uspec
|
||||
# matches by-id link from another uspec
|
||||
uspec1 = {'DEVLINKS': ['/dev/disk/by-path/fakepath',
|
||||
'/dev/disk/by-id/fakeid1',
|
||||
'/dev/disk/by-id/fakeid2']}
|
||||
uspec2 = {'DEVLINKS': ['/dev/disk/by-id/fakeid2',
|
||||
'/dev/disk/by-id/fakeid3']}
|
||||
self.assertTrue(hu.match_device(uspec1, uspec2))
|
||||
|
||||
def test_match_device_wwn(self):
|
||||
# should return true if ID_WWN is given
|
||||
# and if it is the same in both uspecs
|
||||
# and if DEVTYPE is given and if DEVTYPE is disk
|
||||
# or if DEVTYPE is partition and MINOR is the same for both uspecs
|
||||
uspec1 = uspec2 = {'ID_WWN': 'fakewwn',
|
||||
'DEVTYPE': 'disk'}
|
||||
self.assertTrue(hu.match_device(uspec1, uspec2))
|
||||
uspec1 = uspec2 = {'ID_WWN': 'fakewwn',
|
||||
'DEVTYPE': 'partition',
|
||||
'MINOR': '1'}
|
||||
self.assertTrue(hu.match_device(uspec1, uspec2))
|
||||
|
||||
def test_match_device_wwn_false(self):
|
||||
# should return false if ID_WWN is given
|
||||
# and does not match each other
|
||||
uspec1 = {'ID_WWN': 'fakewwn1'}
|
||||
uspec2 = {'ID_WWN': 'fakewwn2'}
|
||||
self.assertFalse(hu.match_device(uspec1, uspec2))
|
||||
|
||||
def test_match_device_devpath(self):
|
||||
# should return true if DEVPATH is given
|
||||
# and if it is the same for both uspecs
|
||||
uspec1 = uspec2 = {'DEVPATH': '/devices/fake'}
|
||||
self.assertTrue(hu.match_device(uspec1, uspec2))
|
||||
|
||||
def test_match_device_serial(self):
|
||||
# should return true if ID_SERIAL_SHORT is given
|
||||
# and if it is the same for both uspecs
|
||||
# and if DEVTYPE is given and if it is 'disk'
|
||||
uspec1 = uspec2 = {'ID_SERIAL_SHORT': 'fakeserial',
|
||||
'DEVTYPE': 'disk'}
|
||||
self.assertTrue(hu.match_device(uspec1, uspec2))
|
||||
|
||||
def test_match_device_serial_false(self):
|
||||
# should return false if ID_SERIAL_SHORT is given
|
||||
# and if it does not match each other
|
||||
uspec1 = {'ID_SERIAL_SHORT': 'fakeserial1'}
|
||||
uspec2 = {'ID_SERIAL_SHORT': 'fakeserial2'}
|
||||
self.assertFalse(hu.match_device(uspec1, uspec2))
|
||||
|
||||
def test_match_device_false(self):
|
||||
uspec1 = {'ID_WWN': 'fakewwn1', 'DEVTYPE': 'disk'}
|
||||
uspec2 = {'ID_WWN': 'fakewwn1', 'DEVTYPE': 'partition'}
|
||||
self.assertFalse(hu.match_device(uspec1, uspec2))
|
|
@ -1,26 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.objects import image
|
||||
|
||||
|
||||
class TestImage(test_base.BaseTestCase):
|
||||
|
||||
def test_unsupported_container(self):
|
||||
self.assertRaises(errors.WrongImageDataError, image.Image, 'uri',
|
||||
'dev', 'format', 'unsupported')
|
|
@ -1,204 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent.drivers import ks_spaces_validator as kssv
|
||||
from fuel_agent import errors
|
||||
|
||||
SAMPLE_SCHEME = [
|
||||
{
|
||||
"name": "sda",
|
||||
"extra": [
|
||||
"disk/by-id/scsi-SATA_VBOX_HARDDISK_VB69050467-b385c7cd",
|
||||
"disk/by-id/ata-VBOX_HARDDISK_VB69050467-b385c7cd"
|
||||
],
|
||||
"free_space": 64907,
|
||||
"volumes": [
|
||||
{
|
||||
"type": "boot",
|
||||
"size": 300
|
||||
},
|
||||
{
|
||||
"mount": "/boot",
|
||||
"size": 200,
|
||||
"type": "raid",
|
||||
"file_system": "ext2",
|
||||
"name": "Boot"
|
||||
},
|
||||
{
|
||||
"type": "lvm_meta_pool",
|
||||
"size": 0
|
||||
},
|
||||
{
|
||||
"size": 19438,
|
||||
"type": "pv",
|
||||
"lvm_meta_size": 64,
|
||||
"vg": "os"
|
||||
},
|
||||
{
|
||||
"size": 45597,
|
||||
"type": "pv",
|
||||
"lvm_meta_size": 64,
|
||||
"vg": "image"
|
||||
}
|
||||
],
|
||||
"type": "disk",
|
||||
"id": "sda",
|
||||
"size": 65535
|
||||
},
|
||||
{
|
||||
"name": "sdb",
|
||||
"extra": [
|
||||
"disk/by-id/scsi-SATA_VBOX_HARDDISK_VBf2923215-708af674",
|
||||
"disk/by-id/ata-VBOX_HARDDISK_VBf2923215-708af674"
|
||||
],
|
||||
"free_space": 64907,
|
||||
"volumes": [
|
||||
{
|
||||
"type": "boot",
|
||||
"size": 300
|
||||
},
|
||||
{
|
||||
"mount": "/boot",
|
||||
"size": 200,
|
||||
"type": "raid",
|
||||
"file_system": "ext2",
|
||||
"name": "Boot"
|
||||
},
|
||||
{
|
||||
"type": "lvm_meta_pool",
|
||||
"size": 64
|
||||
},
|
||||
{
|
||||
"size": 0,
|
||||
"type": "pv",
|
||||
"lvm_meta_size": 0,
|
||||
"vg": "os"
|
||||
},
|
||||
{
|
||||
"size": 64971,
|
||||
"type": "pv",
|
||||
"lvm_meta_size": 64,
|
||||
"vg": "image"
|
||||
}
|
||||
],
|
||||
"type": "disk",
|
||||
"id": "sdb",
|
||||
"size": 65535
|
||||
},
|
||||
{
|
||||
"name": "sdc",
|
||||
"extra": [
|
||||
"disk/by-id/scsi-SATA_VBOX_HARDDISK_VB50ee61eb-84e74fdf",
|
||||
"disk/by-id/ata-VBOX_HARDDISK_VB50ee61eb-84e74fdf"
|
||||
],
|
||||
"free_space": 64907,
|
||||
"volumes": [
|
||||
{
|
||||
"type": "boot",
|
||||
"size": 300
|
||||
},
|
||||
{
|
||||
"mount": "/boot",
|
||||
"size": 200,
|
||||
"type": "raid",
|
||||
"file_system": "ext2",
|
||||
"name": "Boot"
|
||||
},
|
||||
{
|
||||
"type": "lvm_meta_pool",
|
||||
"size": 64
|
||||
},
|
||||
{
|
||||
"size": 0,
|
||||
"type": "pv",
|
||||
"lvm_meta_size": 0,
|
||||
"vg": "os"
|
||||
},
|
||||
{
|
||||
"size": 64971,
|
||||
"type": "pv",
|
||||
"lvm_meta_size": 64,
|
||||
"vg": "image"
|
||||
}
|
||||
],
|
||||
"type": "disk",
|
||||
"id": "disk/by-path/pci-0000:00:0d.0-scsi-0:0:0:0",
|
||||
"size": 65535
|
||||
},
|
||||
{
|
||||
"_allocate_size": "min",
|
||||
"label": "Base System",
|
||||
"min_size": 19374,
|
||||
"volumes": [
|
||||
{
|
||||
"mount": "/",
|
||||
"size": 15360,
|
||||
"type": "lv",
|
||||
"name": "root",
|
||||
"file_system": "ext4"
|
||||
},
|
||||
{
|
||||
"mount": "swap",
|
||||
"size": 4014,
|
||||
"type": "lv",
|
||||
"name": "swap",
|
||||
"file_system": "swap"
|
||||
}
|
||||
],
|
||||
"type": "vg",
|
||||
"id": "os"
|
||||
},
|
||||
{
|
||||
"_allocate_size": "all",
|
||||
"label": "Image Storage",
|
||||
"min_size": 5120,
|
||||
"volumes": [
|
||||
{
|
||||
"mount": "/var/lib/glance",
|
||||
"size": 175347,
|
||||
"type": "lv",
|
||||
"name": "glance",
|
||||
"file_system": "xfs"
|
||||
}
|
||||
],
|
||||
"type": "vg",
|
||||
"id": "image"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class TestKSSpacesValidator(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestKSSpacesValidator, self).setUp()
|
||||
self.fake_scheme = copy.deepcopy(SAMPLE_SCHEME)
|
||||
|
||||
def test_validate_ok(self):
|
||||
kssv.validate(self.fake_scheme)
|
||||
|
||||
def test_validate_jsoschema_fail(self):
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, kssv.validate,
|
||||
[{}])
|
||||
|
||||
def test_validate_no_disks_fail(self):
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, kssv.validate,
|
||||
self.fake_scheme[-2:])
|
||||
|
||||
def test_validate_16T_root_volume_fail(self):
|
||||
self.fake_scheme[3]['volumes'][0]['size'] = 16777216 + 1
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, kssv.validate,
|
||||
self.fake_scheme)
|
|
@ -1,400 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import lvm as lu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
class TestLvmUtils(test_base.BaseTestCase):
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_pvdisplay(self, mock_exec):
|
||||
# should run os command pvdisplay
|
||||
# in order to get actual pv configuration
|
||||
mock_exec.return_value = (
|
||||
'/dev/fake1;vg;892.00m;1024.00m;'
|
||||
'123456-1234-1234-1234-1234-1234-000000\n'
|
||||
'/dev/fake2;;1024.00m;1024.00m;'
|
||||
'123456-1234-1234-1234-1234-1234-111111\n',
|
||||
''
|
||||
)
|
||||
expected = [
|
||||
{
|
||||
'uuid': '123456-1234-1234-1234-1234-1234-000000',
|
||||
'vg': 'vg',
|
||||
'devsize': 1024,
|
||||
'psize': 892,
|
||||
'name': '/dev/fake1',
|
||||
},
|
||||
{
|
||||
'uuid': '123456-1234-1234-1234-1234-1234-111111',
|
||||
'vg': None,
|
||||
'devsize': 1024,
|
||||
'psize': 1024,
|
||||
'name': '/dev/fake2',
|
||||
}
|
||||
]
|
||||
|
||||
pvs = lu.pvdisplay()
|
||||
mock_exec.assert_called_once_with(
|
||||
'pvdisplay',
|
||||
'-C',
|
||||
'--noheading',
|
||||
'--units', 'm',
|
||||
'--options', 'pv_name,vg_name,pv_size,dev_size,pv_uuid',
|
||||
'--separator', ';',
|
||||
check_exit_code=[0]
|
||||
)
|
||||
key = lambda x: x['name']
|
||||
self.assertEqual(sorted(expected, key=key), sorted(pvs, key=key))
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_pvcreate_ok(self, mock_exec, mock_pvdisplay):
|
||||
# should set metadatasize=64 and metadatacopies=2 if they are not set
|
||||
# should run pvcreate command
|
||||
mock_pvdisplay.return_value = [{'name': '/dev/another'}]
|
||||
|
||||
lu.pvcreate('/dev/fake1', metadatasize=32, metadatacopies=1)
|
||||
lu.pvcreate('/dev/fake2', metadatacopies=1)
|
||||
lu.pvcreate('/dev/fake3', metadatasize=32)
|
||||
lu.pvcreate('/dev/fake4')
|
||||
|
||||
expected_calls = [
|
||||
mock.call('pvcreate',
|
||||
'--metadatacopies', '1',
|
||||
'--metadatasize', '32m',
|
||||
'/dev/fake1',
|
||||
check_exit_code=[0]),
|
||||
mock.call('pvcreate',
|
||||
'--metadatacopies', '1',
|
||||
'--metadatasize', '64m',
|
||||
'/dev/fake2',
|
||||
check_exit_code=[0]),
|
||||
mock.call('pvcreate',
|
||||
'--metadatacopies', '2',
|
||||
'--metadatasize', '32m',
|
||||
'/dev/fake3',
|
||||
check_exit_code=[0]),
|
||||
mock.call('pvcreate',
|
||||
'--metadatacopies', '2',
|
||||
'--metadatasize', '64m',
|
||||
'/dev/fake4',
|
||||
check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
def test_pvcreate_duplicate(self, mock_pvdisplay):
|
||||
# should check if pv exists
|
||||
# then raise exception if it exists
|
||||
mock_pvdisplay.return_value = [{'name': '/dev/fake'}]
|
||||
self.assertRaises(
|
||||
errors.PVAlreadyExistsError, lu.pvcreate, '/dev/fake')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_pvremove_ok(self, mock_exec, mock_pvdisplay):
|
||||
# should check if pv exists and is not attached to some vg
|
||||
# then should run pvremove command
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake'}]
|
||||
lu.pvremove('/dev/fake')
|
||||
mock_exec.assert_called_once_with('pvremove', '-ff', '-y', '/dev/fake',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
def test_pvremove_attached_to_vg(self, mock_pvdisplay):
|
||||
# should check if pv exists and is not attached to some vg
|
||||
# then raise exception if it is attached to some vg
|
||||
mock_pvdisplay.return_value = [{'vg': 'some', 'name': '/dev/fake'}]
|
||||
self.assertRaises(errors.PVBelongsToVGError, lu.pvremove, '/dev/fake')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
def test_pvremove_notfound(self, mock_pvdisplay):
|
||||
# should check if pv exists
|
||||
# then should raise exception if it does not exist
|
||||
mock_pvdisplay.return_value = [{'name': '/dev/another'}]
|
||||
self.assertRaises(errors.PVNotFoundError, lu.pvremove, '/dev/fake')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgdisplay(self, mock_exec):
|
||||
# should run os command vgdisplay
|
||||
# in order to get actual vg configuration
|
||||
mock_exec.return_value = (
|
||||
'vg1;123456-1234-1234-1234-1234-1234-000000;2040.00m;2040.00m\n'
|
||||
'vg2;123456-1234-1234-1234-1234-1234-111111;2040.00m;1020.00m\n',
|
||||
''
|
||||
)
|
||||
expected = [
|
||||
{
|
||||
'uuid': '123456-1234-1234-1234-1234-1234-000000',
|
||||
'size': 2040,
|
||||
'free': 2040,
|
||||
'name': 'vg1',
|
||||
},
|
||||
{
|
||||
'uuid': '123456-1234-1234-1234-1234-1234-111111',
|
||||
'size': 2040,
|
||||
'free': 1020,
|
||||
'name': 'vg2',
|
||||
}
|
||||
]
|
||||
|
||||
vg = lu.vgdisplay()
|
||||
mock_exec.assert_called_once_with(
|
||||
'vgdisplay',
|
||||
'-C',
|
||||
'--noheading',
|
||||
'--units', 'm',
|
||||
'--options', 'vg_name,vg_uuid,vg_size,vg_free',
|
||||
'--separator', ';',
|
||||
check_exit_code=[0]
|
||||
)
|
||||
key = lambda x: x['name']
|
||||
self.assertEqual(sorted(expected, key=key), sorted(vg, key=key))
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgcreate_ok(self, mock_exec, mock_vgdisplay, mock_pvdisplay):
|
||||
# should check if vg already exists
|
||||
# should check if all necessary pv exist
|
||||
# should run vgcreate command
|
||||
mock_vgdisplay.return_value = [{'name': 'some'}, {'name': 'another'}]
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake1'},
|
||||
{'vg': None, 'name': '/dev/fake2'}]
|
||||
|
||||
# one pvname
|
||||
lu.vgcreate('vgname', '/dev/fake1')
|
||||
# several pvnames
|
||||
lu.vgcreate('vgname', '/dev/fake1', '/dev/fake2')
|
||||
|
||||
expected_calls = [
|
||||
mock.call('vgcreate', 'vgname', '/dev/fake1',
|
||||
check_exit_code=[0]),
|
||||
mock.call('vgcreate', 'vgname', '/dev/fake1', '/dev/fake2',
|
||||
check_exit_code=[0])
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
def test_vgcreate_duplicate(self, mock_vgdisplay):
|
||||
# should check if vg exists
|
||||
# should raise exception if it exists
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname'}, {'name': 'some'}]
|
||||
self.assertRaises(errors.VGAlreadyExistsError,
|
||||
lu.vgcreate, 'vgname', '/dev/fake')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
def test_vgcreate_pv_not_found(self, mock_vgdisplay, mock_pvdisplay):
|
||||
# should check if all necessary pv exist
|
||||
# should raise error if at least one pv does not
|
||||
mock_vgdisplay.return_value = []
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake1'},
|
||||
{'vg': None, 'name': '/dev/fake2'}]
|
||||
self.assertRaises(errors.PVNotFoundError,
|
||||
lu.vgcreate, 'vgname', '/dev/fake', '/dev/fake2')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
def test_vgcreate_pv_attached(self, mock_vgdisplay, mock_pvdisplay):
|
||||
# should check if all necessary pv are not attached to some vg
|
||||
# should raise error if at least one pv is attached
|
||||
mock_vgdisplay.return_value = []
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake1'},
|
||||
{'vg': 'some', 'name': '/dev/fake2'}]
|
||||
self.assertRaises(errors.PVBelongsToVGError,
|
||||
lu.vgcreate, 'vgname', '/dev/fake1', '/dev/fake2')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgextend_ok(self, mock_exec, mock_vgdisplay, mock_pvdisplay):
|
||||
# should check if vg exists
|
||||
# should check if all necessary pv exist and not attached to any vg
|
||||
# should run vgextend command
|
||||
mock_vgdisplay.return_value = [{'name': 'some'}, {'name': 'another'}]
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake1'},
|
||||
{'vg': None, 'name': '/dev/fake2'}]
|
||||
lu.vgextend('some', '/dev/fake1', '/dev/fake2')
|
||||
mock_exec.assert_called_once_with(
|
||||
'vgextend', 'some', '/dev/fake1', '/dev/fake2',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgextend_not_found(self, mock_exec, mock_vgdisplay):
|
||||
# should check if vg exists
|
||||
# should raise error if it does not
|
||||
mock_vgdisplay.return_value = [{'name': 'some'}]
|
||||
self.assertRaises(errors.VGNotFoundError,
|
||||
lu.vgextend, 'vgname', '/dev/fake1')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
def test_vgextend_pv_not_found(self, mock_vgdisplay, mock_pvdisplay):
|
||||
# should check if all necessary pv exist
|
||||
# should raise error if at least one pv does not
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname'}]
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake1'},
|
||||
{'vg': None, 'name': '/dev/fake2'}]
|
||||
self.assertRaises(errors.PVNotFoundError,
|
||||
lu.vgextend, 'vgname', '/dev/fake', '/dev/fake2')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
def test_vgextend_pv_attached(self, mock_vgdisplay, mock_pvdisplay):
|
||||
# should check if all necessary pv are not attached to some vg
|
||||
# should raise error if at least one pv is attached
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname'}]
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake1'},
|
||||
{'vg': 'some', 'name': '/dev/fake2'}]
|
||||
self.assertRaises(errors.PVBelongsToVGError,
|
||||
lu.vgextend, 'vgname', '/dev/fake1', '/dev/fake2')
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgremove_ok(self, mock_exec, mock_vgdisplay):
|
||||
# should check if vg exists
|
||||
# then run vgremove command if it exists
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname'}, {'name': 'some'}]
|
||||
lu.vgremove('vgname')
|
||||
mock_exec.assert_called_once_with('vgremove', '-f', 'vgname',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgremove_not_found(self, mock_exec, mock_vgdisplay):
|
||||
# should check if vg exists
|
||||
# then raise error if it doesn't
|
||||
mock_vgdisplay.return_value = [{'name': 'some'}]
|
||||
self.assertRaises(errors.VGNotFoundError, lu.vgremove, 'vgname')
|
||||
|
||||
@mock.patch.object(lu, 'lvdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvremove_ok(self, mock_exec, mock_lvdisplay):
|
||||
mock_lvdisplay.return_value = [{'path': '/dev/vg/lv'},
|
||||
{'path': '/dev/vg2/lv2'}]
|
||||
lu.lvremove('/dev/vg/lv')
|
||||
mock_exec.assert_called_once_with('lvremove', '-f', '/dev/vg/lv',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(lu, 'lvdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvremove_not_found(self, mock_exec, mock_lvdisplay):
|
||||
mock_lvdisplay.return_value = [{'path': '/dev/vg/lv'}]
|
||||
self.assertRaises(errors.LVNotFoundError, lu.lvremove, '/dev/vg/lv2')
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(lu, 'lvdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvcreate_ok(self, mock_exec, mock_lvdisplay, mock_vgdisplay):
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname', 'free': 2000},
|
||||
{'name': 'some'}]
|
||||
mock_lvdisplay.return_value = [{'name': 'some'}]
|
||||
lu.lvcreate('vgname', 'lvname', 1000)
|
||||
mock_exec.assert_called_once_with('lvcreate', '--yes', '-L', '1000m',
|
||||
'-n', 'lvname', 'vgname',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvcreate_not_found(self, mock_exec, mock_vgdisplay):
|
||||
mock_vgdisplay.return_value = [{'name': 'some'}]
|
||||
self.assertRaises(errors.VGNotFoundError, lu.lvcreate, 'vgname',
|
||||
'lvname', 1)
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvcreate_not_enough_space(self, mock_exec, mock_vgdisplay):
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname', 'free': 1},
|
||||
{'name': 'some'}]
|
||||
self.assertRaises(errors.NotEnoughSpaceError, lu.lvcreate, 'vgname',
|
||||
'lvname', 2)
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(lu, 'lvdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvcreate_lv_already_exists(self, mock_exec, mock_lvdisplay,
|
||||
mock_vgdisplay):
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname', 'free': 2000},
|
||||
{'name': 'some'}]
|
||||
mock_lvdisplay.return_value = [{'name': 'lvname', 'vg': 'vgname'}]
|
||||
self.assertRaises(errors.LVAlreadyExistsError, lu.lvcreate, 'vgname',
|
||||
'lvname', 1000)
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(lu, 'lvdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvcreate_lv_name_collision(self, mock_exec, mock_lvdisplay,
|
||||
mock_vgdisplay):
|
||||
# lv lvname already exists in another pv
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname', 'free': 2000},
|
||||
{'name': 'some', 'free': 2000}]
|
||||
mock_lvdisplay.return_value = [{'name': 'lvname', 'vg': 'some'}]
|
||||
lu.lvcreate('vgname', 'lvname', 1000)
|
||||
mock_exec.assert_called_once_with('lvcreate', '--yes', '-L', '1000m',
|
||||
'-n', 'lvname', 'vgname',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_lvdisplay(self, mock_exec):
|
||||
mock_exec.return_value = [
|
||||
' lv_name1;1234.12m;vg_name;lv_uuid1\n'
|
||||
' lv_name2;5678.79m;vg_name;lv_uuid2\n ']
|
||||
expected_lvs = [{'name': 'lv_name1', 'size': 1235, 'vg': 'vg_name',
|
||||
'uuid': 'lv_uuid1', 'path': '/dev/vg_name/lv_name1'},
|
||||
{'name': 'lv_name2', 'size': 5679, 'vg': 'vg_name',
|
||||
'uuid': 'lv_uuid2', 'path': '/dev/vg_name/lv_name2'}]
|
||||
actual_lvs = lu.lvdisplay()
|
||||
self.assertEqual(expected_lvs, actual_lvs)
|
||||
mock_exec.assert_called_once_with('lvdisplay', '-C', '--noheading',
|
||||
'--units', 'm', '--options',
|
||||
'lv_name,lv_size,vg_name,lv_uuid',
|
||||
'--separator', ';',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgreduce_ok(self, mock_exec, mock_vgdisplay, mock_pvdisplay):
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname'}, {'name': 'some'}]
|
||||
mock_pvdisplay.return_value = [{'vg': 'vgname', 'name': '/dev/fake1'},
|
||||
{'vg': 'vgname', 'name': '/dev/fake2'}]
|
||||
lu.vgreduce('vgname', '/dev/fake1', '/dev/fake2')
|
||||
mock_exec.assert_called_once_with('vgreduce', '-f', 'vgname',
|
||||
'/dev/fake1', '/dev/fake2',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
def test_vgreduce_vg_not_found(self, mock_vgdisplay):
|
||||
mock_vgdisplay.return_value = [{'name': 'some'}]
|
||||
self.assertRaises(errors.VGNotFoundError, lu.vgreduce, 'vgname1',
|
||||
'/dev/fake1', '/dev/fake2')
|
||||
|
||||
@mock.patch.object(lu, 'pvdisplay')
|
||||
@mock.patch.object(lu, 'vgdisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_vgreduce_pv_not_attached(self, mock_exec, mock_vgdisplay,
|
||||
mock_pvdisplay):
|
||||
mock_vgdisplay.return_value = [{'name': 'vgname'}, {'name': 'some'}]
|
||||
mock_pvdisplay.return_value = [{'vg': None, 'name': '/dev/fake1'},
|
||||
{'vg': None, 'name': '/dev/fake2'}]
|
||||
self.assertRaises(errors.PVNotFoundError, lu.vgreduce, 'vgname',
|
||||
'/dev/fake1', '/dev/fake2')
|
|
@ -1,532 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
import os
|
||||
import signal
|
||||
|
||||
from oslo.config import cfg
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent import manager
|
||||
from fuel_agent import objects
|
||||
from fuel_agent.objects import partition
|
||||
from fuel_agent.tests import test_nailgun
|
||||
from fuel_agent.utils import artifact as au
|
||||
from fuel_agent.utils import fs as fu
|
||||
from fuel_agent.utils import hardware as hu
|
||||
from fuel_agent.utils import lvm as lu
|
||||
from fuel_agent.utils import md as mu
|
||||
from fuel_agent.utils import partition as pu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class TestManager(test_base.BaseTestCase):
|
||||
|
||||
@mock.patch('yaml.load')
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def setUp(self, mock_lbd, mock_http, mock_yaml):
|
||||
super(TestManager, self).setUp()
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
self.mgr = manager.Manager(test_nailgun.PROVISION_SAMPLE_DATA)
|
||||
|
||||
@mock.patch('six.moves.builtins.open')
|
||||
@mock.patch.object(os, 'symlink')
|
||||
@mock.patch.object(os, 'remove')
|
||||
@mock.patch.object(os, 'path')
|
||||
@mock.patch.object(os, 'listdir')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(mu, 'mdclean_all')
|
||||
@mock.patch.object(lu, 'lvremove_all')
|
||||
@mock.patch.object(lu, 'vgremove_all')
|
||||
@mock.patch.object(lu, 'pvremove_all')
|
||||
@mock.patch.object(fu, 'make_fs')
|
||||
@mock.patch.object(lu, 'lvcreate')
|
||||
@mock.patch.object(lu, 'vgcreate')
|
||||
@mock.patch.object(lu, 'pvcreate')
|
||||
@mock.patch.object(mu, 'mdcreate')
|
||||
@mock.patch.object(pu, 'set_gpt_type')
|
||||
@mock.patch.object(pu, 'set_partition_flag')
|
||||
@mock.patch.object(pu, 'make_partition')
|
||||
@mock.patch.object(pu, 'make_label')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def test_do_partitioning(self, mock_hu_lbd, mock_pu_ml, mock_pu_mp,
|
||||
mock_pu_spf, mock_pu_sgt, mock_mu_m, mock_lu_p,
|
||||
mock_lu_v, mock_lu_l, mock_fu_mf, mock_pvr,
|
||||
mock_vgr, mock_lvr, mock_mdr, mock_exec,
|
||||
mock_os_ld, mock_os_p, mock_os_r, mock_os_s,
|
||||
mock_open):
|
||||
mock_os_ld.return_value = ['not_a_rule', 'fake.rules']
|
||||
mock_os_p.exists.return_value = True
|
||||
mock_hu_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
self.mgr.do_partitioning()
|
||||
mock_pu_ml_expected_calls = [mock.call('/dev/sda', 'gpt'),
|
||||
mock.call('/dev/sdb', 'gpt'),
|
||||
mock.call('/dev/sdc', 'gpt')]
|
||||
self.assertEqual(mock_pu_ml_expected_calls, mock_pu_ml.call_args_list)
|
||||
|
||||
mock_pu_mp_expected_calls = [
|
||||
mock.call('/dev/sda', 1, 25, 'primary'),
|
||||
mock.call('/dev/sda', 25, 225, 'primary'),
|
||||
mock.call('/dev/sda', 225, 425, 'primary'),
|
||||
mock.call('/dev/sda', 425, 625, 'primary'),
|
||||
mock.call('/dev/sda', 625, 20063, 'primary'),
|
||||
mock.call('/dev/sda', 20063, 65660, 'primary'),
|
||||
mock.call('/dev/sda', 65660, 65680, 'primary'),
|
||||
mock.call('/dev/sdb', 1, 25, 'primary'),
|
||||
mock.call('/dev/sdb', 25, 225, 'primary'),
|
||||
mock.call('/dev/sdb', 225, 65196, 'primary'),
|
||||
mock.call('/dev/sdc', 1, 25, 'primary'),
|
||||
mock.call('/dev/sdc', 25, 225, 'primary'),
|
||||
mock.call('/dev/sdc', 225, 65196, 'primary')]
|
||||
self.assertEqual(mock_pu_mp_expected_calls, mock_pu_mp.call_args_list)
|
||||
|
||||
mock_pu_spf_expected_calls = [mock.call('/dev/sda', 1, 'bios_grub'),
|
||||
mock.call('/dev/sdb', 1, 'bios_grub'),
|
||||
mock.call('/dev/sdc', 1, 'bios_grub')]
|
||||
self.assertEqual(mock_pu_spf_expected_calls,
|
||||
mock_pu_spf.call_args_list)
|
||||
|
||||
mock_pu_sgt_expected_calls = [mock.call('/dev/sda', 4, 'fake_guid')]
|
||||
self.assertEqual(mock_pu_sgt_expected_calls,
|
||||
mock_pu_sgt.call_args_list)
|
||||
|
||||
mock_lu_p_expected_calls = [
|
||||
mock.call('/dev/sda5', metadatasize=28, metadatacopies=2),
|
||||
mock.call('/dev/sda6', metadatasize=28, metadatacopies=2),
|
||||
mock.call('/dev/sdb3', metadatasize=28, metadatacopies=2),
|
||||
mock.call('/dev/sdc3', metadatasize=28, metadatacopies=2)]
|
||||
self.assertEqual(mock_lu_p_expected_calls, mock_lu_p.call_args_list)
|
||||
|
||||
mock_lu_v_expected_calls = [mock.call('os', '/dev/sda5'),
|
||||
mock.call('image', '/dev/sda6',
|
||||
'/dev/sdb3', '/dev/sdc3')]
|
||||
self.assertEqual(mock_lu_v_expected_calls, mock_lu_v.call_args_list)
|
||||
|
||||
mock_lu_l_expected_calls = [mock.call('os', 'root', 15360),
|
||||
mock.call('os', 'swap', 4014),
|
||||
mock.call('image', 'glance', 175347)]
|
||||
self.assertEqual(mock_lu_l_expected_calls, mock_lu_l.call_args_list)
|
||||
|
||||
mock_fu_mf_expected_calls = [
|
||||
mock.call('ext2', '', '', '/dev/sda3'),
|
||||
mock.call('ext2', '', '', '/dev/sda4'),
|
||||
mock.call('swap', '', '', '/dev/mapper/os-swap'),
|
||||
mock.call('xfs', '', '', '/dev/mapper/image-glance')]
|
||||
self.assertEqual(mock_fu_mf_expected_calls, mock_fu_mf.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'calculate_md5')
|
||||
@mock.patch('os.path.getsize')
|
||||
@mock.patch('yaml.load')
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(utils, 'render_and_save')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def test_do_configdrive(self, mock_lbd, mock_u_ras, mock_u_e,
|
||||
mock_http_req, mock_yaml, mock_get_size, mock_md5):
|
||||
mock_get_size.return_value = 123
|
||||
mock_md5.return_value = 'fakemd5'
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
self.assertEqual(1, len(self.mgr.driver.image_scheme.images))
|
||||
self.mgr.do_configdrive()
|
||||
mock_u_ras_expected_calls = [
|
||||
mock.call(CONF.nc_template_path,
|
||||
['cloud_config_pro_fi-le.jinja2',
|
||||
'cloud_config_pro.jinja2',
|
||||
'cloud_config_pro_fi.jinja2',
|
||||
'cloud_config.jinja2'],
|
||||
mock.ANY, '%s/%s' % (CONF.tmp_path, 'cloud_config.txt')),
|
||||
mock.call(CONF.nc_template_path,
|
||||
['boothook_pro_fi-le.jinja2',
|
||||
'boothook_pro.jinja2',
|
||||
'boothook_pro_fi.jinja2',
|
||||
'boothook.jinja2'],
|
||||
mock.ANY, '%s/%s' % (CONF.tmp_path, 'boothook.txt')),
|
||||
mock.call(CONF.nc_template_path,
|
||||
['meta-data_pro_fi-le.jinja2',
|
||||
'meta-data_pro.jinja2',
|
||||
'meta-data_pro_fi.jinja2',
|
||||
'meta-data.jinja2'],
|
||||
mock.ANY, '%s/%s' % (CONF.tmp_path, 'meta-data'))]
|
||||
self.assertEqual(mock_u_ras_expected_calls, mock_u_ras.call_args_list)
|
||||
|
||||
mock_u_e_expected_calls = [
|
||||
mock.call('write-mime-multipart',
|
||||
'--output=%s' % ('%s/%s' % (CONF.tmp_path, 'user-data')),
|
||||
'%s:text/cloud-boothook' % ('%s/%s' % (CONF.tmp_path,
|
||||
'boothook.txt')),
|
||||
'%s:text/cloud-config' % ('%s/%s' % (CONF.tmp_path,
|
||||
'cloud_config.txt'))
|
||||
),
|
||||
mock.call('genisoimage', '-output', CONF.config_drive_path,
|
||||
'-volid', 'cidata', '-joliet', '-rock',
|
||||
'%s/%s' % (CONF.tmp_path, 'user-data'),
|
||||
'%s/%s' % (CONF.tmp_path, 'meta-data'))]
|
||||
self.assertEqual(mock_u_e_expected_calls, mock_u_e.call_args_list)
|
||||
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
|
||||
cf_drv_img = self.mgr.driver.image_scheme.images[-1]
|
||||
self.assertEqual('file://%s' % CONF.config_drive_path, cf_drv_img.uri)
|
||||
self.assertEqual('/dev/sda7',
|
||||
self.mgr.driver.partition_scheme.configdrive_device())
|
||||
self.assertEqual('iso9660', cf_drv_img.format)
|
||||
self.assertEqual('raw', cf_drv_img.container)
|
||||
self.assertEqual('fakemd5', cf_drv_img.md5)
|
||||
self.assertEqual(123, cf_drv_img.size)
|
||||
|
||||
@mock.patch('yaml.load')
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
@mock.patch.object(partition.PartitionScheme, 'configdrive_device')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(utils, 'render_and_save')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def test_do_configdrive_no_configdrive_device(self, mock_lbd, mock_u_ras,
|
||||
mock_u_e, mock_p_ps_cd,
|
||||
mock_http_req, mock_yaml):
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
mock_p_ps_cd.return_value = None
|
||||
self.assertRaises(errors.WrongPartitionSchemeError,
|
||||
self.mgr.do_configdrive)
|
||||
|
||||
@mock.patch.object(utils, 'calculate_md5')
|
||||
@mock.patch('os.path.getsize')
|
||||
@mock.patch('yaml.load')
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
@mock.patch.object(fu, 'extend_fs')
|
||||
@mock.patch.object(au, 'GunzipStream')
|
||||
@mock.patch.object(au, 'LocalFile')
|
||||
@mock.patch.object(au, 'HttpUrl')
|
||||
@mock.patch.object(au, 'Chain')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(utils, 'render_and_save')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def test_do_copyimage(self, mock_lbd, mock_u_ras, mock_u_e, mock_au_c,
|
||||
mock_au_h, mock_au_l, mock_au_g, mock_fu_ef,
|
||||
mock_http_req, mock_yaml, mock_get_size, mock_md5):
|
||||
|
||||
class FakeChain(object):
|
||||
processors = []
|
||||
|
||||
def append(self, thing):
|
||||
self.processors.append(thing)
|
||||
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
mock_au_c.return_value = FakeChain()
|
||||
self.mgr.do_configdrive()
|
||||
self.mgr.do_copyimage()
|
||||
imgs = self.mgr.driver.image_scheme.images
|
||||
self.assertEqual(2, len(imgs))
|
||||
expected_processors_list = []
|
||||
for img in imgs[:-1]:
|
||||
expected_processors_list += [
|
||||
img.uri,
|
||||
au.HttpUrl,
|
||||
au.GunzipStream,
|
||||
img.target_device
|
||||
]
|
||||
expected_processors_list += [
|
||||
imgs[-1].uri,
|
||||
au.LocalFile,
|
||||
imgs[-1].target_device
|
||||
]
|
||||
self.assertEqual(expected_processors_list,
|
||||
mock_au_c.return_value.processors)
|
||||
mock_fu_ef_expected_calls = [
|
||||
mock.call('ext4', '/dev/mapper/os-root')]
|
||||
self.assertEqual(mock_fu_ef_expected_calls, mock_fu_ef.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'calculate_md5')
|
||||
@mock.patch('os.path.getsize')
|
||||
@mock.patch('yaml.load')
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
@mock.patch.object(fu, 'extend_fs')
|
||||
@mock.patch.object(au, 'GunzipStream')
|
||||
@mock.patch.object(au, 'LocalFile')
|
||||
@mock.patch.object(au, 'HttpUrl')
|
||||
@mock.patch.object(au, 'Chain')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(utils, 'render_and_save')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def test_do_copyimage_md5_matches(self, mock_lbd, mock_u_ras, mock_u_e,
|
||||
mock_au_c, mock_au_h, mock_au_l,
|
||||
mock_au_g, mock_fu_ef, mock_http_req,
|
||||
mock_yaml, mock_get_size, mock_md5):
|
||||
|
||||
class FakeChain(object):
|
||||
processors = []
|
||||
|
||||
def append(self, thing):
|
||||
self.processors.append(thing)
|
||||
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
mock_get_size.return_value = 123
|
||||
mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5']
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
mock_au_c.return_value = FakeChain()
|
||||
self.mgr.driver.image_scheme.images[0].size = 1234
|
||||
self.mgr.driver.image_scheme.images[0].md5 = 'really_fakemd5'
|
||||
self.mgr.do_configdrive()
|
||||
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
|
||||
self.mgr.do_copyimage()
|
||||
expected_md5_calls = [mock.call('/tmp/config-drive.img', 123),
|
||||
mock.call('/dev/mapper/os-root', 1234),
|
||||
mock.call('/dev/sda7', 123)]
|
||||
self.assertEqual(expected_md5_calls, mock_md5.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'calculate_md5')
|
||||
@mock.patch('os.path.getsize')
|
||||
@mock.patch('yaml.load')
|
||||
@mock.patch.object(utils, 'init_http_request')
|
||||
@mock.patch.object(fu, 'extend_fs')
|
||||
@mock.patch.object(au, 'GunzipStream')
|
||||
@mock.patch.object(au, 'LocalFile')
|
||||
@mock.patch.object(au, 'HttpUrl')
|
||||
@mock.patch.object(au, 'Chain')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(utils, 'render_and_save')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
def test_do_copyimage_md5_mismatch(self, mock_lbd, mock_u_ras, mock_u_e,
|
||||
mock_au_c, mock_au_h, mock_au_l,
|
||||
mock_au_g, mock_fu_ef, mock_http_req,
|
||||
mock_yaml, mock_get_size, mock_md5):
|
||||
|
||||
class FakeChain(object):
|
||||
processors = []
|
||||
|
||||
def append(self, thing):
|
||||
self.processors.append(thing)
|
||||
|
||||
def process(self):
|
||||
pass
|
||||
|
||||
mock_get_size.return_value = 123
|
||||
mock_md5.side_effect = ['fakemd5', 'really_fakemd5', 'fakemd5']
|
||||
mock_lbd.return_value = test_nailgun.LIST_BLOCK_DEVICES_SAMPLE
|
||||
mock_au_c.return_value = FakeChain()
|
||||
self.mgr.driver.image_scheme.images[0].size = 1234
|
||||
self.mgr.driver.image_scheme.images[0].md5 = 'fakemd5'
|
||||
self.mgr.do_configdrive()
|
||||
self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
|
||||
self.assertRaises(errors.ImageChecksumMismatchError,
|
||||
self.mgr.do_copyimage)
|
||||
|
||||
@mock.patch('fuel_agent.manager.bu', create=True)
|
||||
@mock.patch('fuel_agent.manager.fu', create=True)
|
||||
@mock.patch('fuel_agent.manager.utils', create=True)
|
||||
@mock.patch('fuel_agent.manager.os', create=True)
|
||||
@mock.patch('fuel_agent.manager.shutil.move')
|
||||
@mock.patch('fuel_agent.manager.open',
|
||||
create=True, new_callable=mock.mock_open)
|
||||
@mock.patch('fuel_agent.manager.tempfile.mkdtemp')
|
||||
@mock.patch('fuel_agent.manager.yaml.safe_dump')
|
||||
@mock.patch.object(manager.Manager, 'mount_target')
|
||||
@mock.patch.object(manager.Manager, 'umount_target')
|
||||
def test_do_build_image(self, mock_umount_target, mock_mount_target,
|
||||
mock_yaml_dump, mock_mkdtemp,
|
||||
mock_open, mock_shutil_move, mock_os,
|
||||
mock_utils, mock_fu, mock_bu):
|
||||
|
||||
loops = [objects.Loop(), objects.Loop()]
|
||||
|
||||
self.mgr.driver.image_scheme = objects.ImageScheme([
|
||||
objects.Image('file:///fake/img.img.gz', loops[0], 'ext4', 'gzip'),
|
||||
objects.Image('file:///fake/img-boot.img.gz',
|
||||
loops[1], 'ext2', 'gzip')])
|
||||
self.mgr.driver.partition_scheme = objects.PartitionScheme()
|
||||
self.mgr.driver.partition_scheme.add_fs(
|
||||
device=loops[0], mount='/', fs_type='ext4')
|
||||
self.mgr.driver.partition_scheme.add_fs(
|
||||
device=loops[1], mount='/boot', fs_type='ext2')
|
||||
self.mgr.driver.metadata_uri = 'file:///fake/img.yaml'
|
||||
self.mgr.driver.operating_system = objects.Ubuntu(
|
||||
repos=[
|
||||
objects.DEBRepo('ubuntu', 'http://fakeubuntu',
|
||||
'trusty', 'fakesection', priority=900),
|
||||
objects.DEBRepo('ubuntu_zero', 'http://fakeubuntu_zero',
|
||||
'trusty', 'fakesection', priority=None),
|
||||
objects.DEBRepo('mos', 'http://fakemos',
|
||||
'mosX.Y', 'fakesection', priority=1000)],
|
||||
packages=['fakepackage1', 'fakepackage2'])
|
||||
|
||||
mock_os.path.exists.return_value = False
|
||||
mock_os.path.join.return_value = '/tmp/imgdir/proc'
|
||||
mock_os.path.basename.side_effect = ['img.img.gz', 'img-boot.img.gz']
|
||||
mock_bu.create_sparse_tmp_file.side_effect = \
|
||||
['/tmp/img', '/tmp/img-boot']
|
||||
mock_bu.get_free_loop_device.side_effect = ['/dev/loop0', '/dev/loop1']
|
||||
mock_mkdtemp.return_value = '/tmp/imgdir'
|
||||
getsize_side = [20, 2, 10, 1]
|
||||
mock_os.path.getsize.side_effect = getsize_side
|
||||
md5_side = ['fakemd5_raw', 'fakemd5_gzip',
|
||||
'fakemd5_raw_boot', 'fakemd5_gzip_boot']
|
||||
mock_utils.calculate_md5.side_effect = md5_side
|
||||
mock_bu.containerize.side_effect = ['/tmp/img.gz', '/tmp/img-boot.gz']
|
||||
mock_bu.stop_chrooted_processes.side_effect = [
|
||||
False, True, False, True]
|
||||
|
||||
self.mgr.do_build_image()
|
||||
self.assertEqual(
|
||||
[mock.call('/fake/img.img.gz'),
|
||||
mock.call('/fake/img-boot.img.gz')],
|
||||
mock_os.path.exists.call_args_list)
|
||||
self.assertEqual([mock.call(dir=CONF.image_build_dir,
|
||||
suffix=CONF.image_build_suffix)] * 2,
|
||||
mock_bu.create_sparse_tmp_file.call_args_list)
|
||||
self.assertEqual([mock.call()] * 2,
|
||||
mock_bu.get_free_loop_device.call_args_list)
|
||||
self.assertEqual([mock.call('/tmp/img', '/dev/loop0'),
|
||||
mock.call('/tmp/img-boot', '/dev/loop1')],
|
||||
mock_bu.attach_file_to_loop.call_args_list)
|
||||
self.assertEqual([mock.call(fs_type='ext4', fs_options='',
|
||||
fs_label='', dev='/dev/loop0'),
|
||||
mock.call(fs_type='ext2', fs_options='',
|
||||
fs_label='', dev='/dev/loop1')],
|
||||
mock_fu.make_fs.call_args_list)
|
||||
mock_mkdtemp.assert_called_once_with(dir=CONF.image_build_dir,
|
||||
suffix=CONF.image_build_suffix)
|
||||
mock_mount_target.assert_called_once_with(
|
||||
'/tmp/imgdir', treat_mtab=False, pseudo=False)
|
||||
self.assertEqual([mock.call('/tmp/imgdir')] * 2,
|
||||
mock_bu.suppress_services_start.call_args_list)
|
||||
mock_bu.run_debootstrap.assert_called_once_with(
|
||||
uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir')
|
||||
mock_bu.set_apt_get_env.assert_called_once_with()
|
||||
mock_bu.pre_apt_get.assert_called_once_with('/tmp/imgdir')
|
||||
self.assertEqual([
|
||||
mock.call(name='ubuntu',
|
||||
uri='http://fakeubuntu',
|
||||
suite='trusty',
|
||||
section='fakesection',
|
||||
chroot='/tmp/imgdir'),
|
||||
mock.call(name='ubuntu_zero',
|
||||
uri='http://fakeubuntu_zero',
|
||||
suite='trusty',
|
||||
section='fakesection',
|
||||
chroot='/tmp/imgdir'),
|
||||
mock.call(name='mos',
|
||||
uri='http://fakemos',
|
||||
suite='mosX.Y',
|
||||
section='fakesection',
|
||||
chroot='/tmp/imgdir')],
|
||||
mock_bu.add_apt_source.call_args_list)
|
||||
|
||||
# we don't call add_apt_preference for ubuntu_zero
|
||||
# because it has priority == None
|
||||
self.assertEqual([
|
||||
mock.call(name='ubuntu',
|
||||
priority=900,
|
||||
suite='trusty',
|
||||
section='fakesection',
|
||||
chroot='/tmp/imgdir',
|
||||
uri='http://fakeubuntu'),
|
||||
mock.call(name='mos',
|
||||
priority=1000,
|
||||
suite='mosX.Y',
|
||||
section='fakesection',
|
||||
chroot='/tmp/imgdir',
|
||||
uri='http://fakemos')],
|
||||
mock_bu.add_apt_preference.call_args_list)
|
||||
mock_utils.makedirs_if_not_exists.assert_called_once_with(
|
||||
'/tmp/imgdir/proc')
|
||||
self.assertEqual([
|
||||
mock.call('tune2fs', '-O', '^has_journal', '/dev/loop0'),
|
||||
mock.call('tune2fs', '-O', 'has_journal', '/dev/loop0')],
|
||||
mock_utils.execute.call_args_list)
|
||||
mock_fu.mount_bind.assert_called_once_with('/tmp/imgdir', '/proc')
|
||||
mock_bu.run_apt_get.assert_called_once_with(
|
||||
'/tmp/imgdir', packages=['fakepackage1', 'fakepackage2'])
|
||||
mock_bu.do_post_inst.assert_called_once_with('/tmp/imgdir')
|
||||
signal_calls = mock_bu.stop_chrooted_processes.call_args_list
|
||||
self.assertEqual(2 * [mock.call('/tmp/imgdir', signal=signal.SIGTERM),
|
||||
mock.call('/tmp/imgdir', signal=signal.SIGKILL)],
|
||||
signal_calls)
|
||||
self.assertEqual(
|
||||
[mock.call('/tmp/imgdir/proc')] * 2,
|
||||
mock_fu.umount_fs.call_args_list)
|
||||
self.assertEqual(
|
||||
[mock.call(
|
||||
'/tmp/imgdir', try_lazy_umount=False, pseudo=False)] * 2,
|
||||
mock_umount_target.call_args_list)
|
||||
self.assertEqual(
|
||||
[mock.call('/dev/loop0'), mock.call('/dev/loop1')] * 2,
|
||||
mock_bu.deattach_loop.call_args_list)
|
||||
self.assertEqual([mock.call('/tmp/img'), mock.call('/tmp/img-boot')],
|
||||
mock_bu.shrink_sparse_file.call_args_list)
|
||||
self.assertEqual([mock.call('/tmp/img'),
|
||||
mock.call('/fake/img.img.gz'),
|
||||
mock.call('/tmp/img-boot'),
|
||||
mock.call('/fake/img-boot.img.gz')],
|
||||
mock_os.path.getsize.call_args_list)
|
||||
self.assertEqual([mock.call('/tmp/img', 20),
|
||||
mock.call('/fake/img.img.gz', 2),
|
||||
mock.call('/tmp/img-boot', 10),
|
||||
mock.call('/fake/img-boot.img.gz', 1)],
|
||||
mock_utils.calculate_md5.call_args_list)
|
||||
self.assertEqual([mock.call('/tmp/img', 'gzip'),
|
||||
mock.call('/tmp/img-boot', 'gzip')],
|
||||
mock_bu.containerize.call_args_list)
|
||||
mock_open.assert_called_once_with('/fake/img.yaml', 'w')
|
||||
self.assertEqual(
|
||||
[mock.call('/tmp/img.gz', '/fake/img.img.gz'),
|
||||
mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')],
|
||||
mock_shutil_move.call_args_list)
|
||||
|
||||
metadata = {}
|
||||
for repo in self.mgr.driver.operating_system.repos:
|
||||
metadata.setdefault('repos', []).append({
|
||||
'type': 'deb',
|
||||
'name': repo.name,
|
||||
'uri': repo.uri,
|
||||
'suite': repo.suite,
|
||||
'section': repo.section,
|
||||
'priority': repo.priority,
|
||||
'meta': repo.meta})
|
||||
metadata['packages'] = self.mgr.driver.operating_system.packages
|
||||
metadata['images'] = [
|
||||
{
|
||||
'raw_md5': md5_side[0],
|
||||
'raw_size': getsize_side[0],
|
||||
'raw_name': None,
|
||||
'container_name':
|
||||
os.path.basename(
|
||||
self.mgr.driver.image_scheme.images[0].uri.split(
|
||||
'file://', 1)[1]),
|
||||
'container_md5': md5_side[1],
|
||||
'container_size': getsize_side[1],
|
||||
'container': self.mgr.driver.image_scheme.images[0].container,
|
||||
'format': self.mgr.driver.image_scheme.images[0].format
|
||||
},
|
||||
{
|
||||
'raw_md5': md5_side[2],
|
||||
'raw_size': getsize_side[2],
|
||||
'raw_name': None,
|
||||
'container_name':
|
||||
os.path.basename(
|
||||
self.mgr.driver.image_scheme.images[1].uri.split(
|
||||
'file://', 1)[1]),
|
||||
'container_md5': md5_side[3],
|
||||
'container_size': getsize_side[3],
|
||||
'container': self.mgr.driver.image_scheme.images[1].container,
|
||||
'format': self.mgr.driver.image_scheme.images[1].format
|
||||
}
|
||||
]
|
||||
mock_yaml_dump.assert_called_once_with(metadata, stream=mock_open())
|
|
@ -1,277 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
import six
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import hardware as hu
|
||||
from fuel_agent.utils import md as mu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
if six.PY2:
|
||||
OPEN_FUNCTION_NAME = '__builtin__.open'
|
||||
else:
|
||||
OPEN_FUNCTION_NAME = 'builtins.open'
|
||||
|
||||
|
||||
class TestMdUtils(test_base.BaseTestCase):
|
||||
|
||||
@mock.patch('fuel_agent.utils.md.utils.execute')
|
||||
def test_mddisplay_nostate_detail(self, mock_exec):
|
||||
mock_exec.return_value = (
|
||||
"""/dev/md127:
|
||||
Version : imsm
|
||||
Raid Level : container
|
||||
Total Devices : 2
|
||||
|
||||
Working Devices : 2
|
||||
|
||||
|
||||
UUID : 46a4fc60:21554de1:1edfad0f:c137ddac
|
||||
Member Arrays :
|
||||
|
||||
Number Major Minor RaidDevice
|
||||
|
||||
0 8 0 - /dev/sda
|
||||
1 8 16 - /dev/sdb""",
|
||||
''
|
||||
)
|
||||
|
||||
expected = [{
|
||||
'Raid Level': 'container',
|
||||
'UUID': '46a4fc60:21554de1:1edfad0f:c137ddac',
|
||||
'Version': 'imsm',
|
||||
'devices': ['/dev/sda', '/dev/sdb'],
|
||||
'name': '/dev/md127',
|
||||
}]
|
||||
|
||||
mds = mu.mddisplay(['/dev/md127'])
|
||||
mock_exec.assert_called_once_with(
|
||||
'mdadm', '--detail', '/dev/md127', check_exit_code=[0])
|
||||
|
||||
self.assertItemsEqual(expected, mds)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_mddisplay(self, mock_exec):
|
||||
# should read file /proc/mdstat
|
||||
# should get detailed description for all md devices
|
||||
# should return list of dicts representing md devices
|
||||
|
||||
mock_open_data = """Personalities : [raid1]
|
||||
md0 : active raid1 loop5[1] loop4[0]
|
||||
102272 blocks super 1.2 [2/2] [UU]
|
||||
|
||||
unused devices: <none>
|
||||
|
||||
|
||||
"""
|
||||
mock_open = mock.mock_open(read_data=mock_open_data)
|
||||
patcher = mock.patch(OPEN_FUNCTION_NAME, new=mock_open)
|
||||
patcher.start()
|
||||
|
||||
mock_exec.return_value = (
|
||||
"""/dev/md0:
|
||||
Version : 1.2
|
||||
Creation Time : Wed Jun 18 18:44:57 2014
|
||||
Raid Level : raid1
|
||||
Array Size : 102272 (99.89 MiB 104.73 MB)
|
||||
Used Dev Size : 102272 (99.89 MiB 104.73 MB)
|
||||
Raid Devices : 2
|
||||
Total Devices : 2
|
||||
Persistence : Superblock is persistent
|
||||
|
||||
Update Time : Wed Jun 18 18:45:01 2014
|
||||
State : clean
|
||||
Active Devices : 2
|
||||
Working Devices : 2
|
||||
Failed Devices : 0
|
||||
Spare Devices : 0
|
||||
|
||||
Name : localhost.localdomain:0 (local to host
|
||||
localhost.localdomain)
|
||||
UUID : 12dd4cfc:6b2ac9db:94564538:a6ffee82
|
||||
Events : 17
|
||||
|
||||
Number Major Minor RaidDevice State
|
||||
0 7 4 0 active sync /dev/loop4
|
||||
1 7 5 1 active sync /dev/loop5""",
|
||||
''
|
||||
)
|
||||
|
||||
expected = [{
|
||||
'name': '/dev/md0',
|
||||
'Version': '1.2',
|
||||
'Raid Level': 'raid1',
|
||||
'Raid Devices': '2',
|
||||
'Active Devices': '2',
|
||||
'Spare Devices': '0',
|
||||
'Failed Devices': '0',
|
||||
'State': 'clean',
|
||||
'UUID': '12dd4cfc:6b2ac9db:94564538:a6ffee82',
|
||||
'devices': ['/dev/loop4', '/dev/loop5']
|
||||
}]
|
||||
|
||||
mds = mu.mddisplay()
|
||||
mock_exec.assert_called_once_with(
|
||||
'mdadm', '--detail', '/dev/md0', check_exit_code=[0])
|
||||
|
||||
key = lambda x: x['name']
|
||||
self.assertEqual(sorted(expected, key=key), sorted(mds, key=key))
|
||||
patcher.stop()
|
||||
|
||||
@mock.patch.object(mu, 'mdclean')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_mdcreate_ok(self, mock_exec, mock_mddisplay,
|
||||
mock_bdevs, mock_mdclean):
|
||||
# should check if md already exists
|
||||
# should check if md level is valid
|
||||
# should check if all necessary devices exist
|
||||
# should check if all devices are not parts of some md
|
||||
# should clean md metadata which possibly are on all devices
|
||||
# should run mdadm command to create new md
|
||||
|
||||
mock_mddisplay.return_value = \
|
||||
[{'name': '/dev/md10', 'devices': ['/dev/fake10']},
|
||||
# should also accept devices with missing 'devices' entry
|
||||
{'name': '/dev/md11'}]
|
||||
mock_bdevs.return_value = [{'device': '/dev/fake1'},
|
||||
{'device': '/dev/fake2'}]
|
||||
|
||||
mu.mdcreate('/dev/md0', 'mirror', '/dev/fake1', '/dev/fake2')
|
||||
mock_mdclean_expected_calls = [mock.call('/dev/fake1'),
|
||||
mock.call('/dev/fake2')]
|
||||
self.assertEqual(mock_mdclean_expected_calls,
|
||||
mock_mdclean.call_args_list)
|
||||
mock_exec.assert_called_once_with(
|
||||
'mdadm', '--create', '--force', '/dev/md0', '-e0.90',
|
||||
'--level=mirror',
|
||||
'--raid-devices=2', '/dev/fake1', '/dev/fake2',
|
||||
check_exit_code=[0])
|
||||
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
def test_mdcreate_duplicate(self, mock_mddisplay):
|
||||
# should check if md already exists
|
||||
# should raise error if it exists
|
||||
mock_mddisplay.return_value = [{'name': '/dev/md0'}]
|
||||
self.assertRaises(
|
||||
errors.MDAlreadyExistsError, mu.mdcreate,
|
||||
'/dev/md0', 'mirror', '/dev/fake')
|
||||
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
def test_mdcreate_unsupported_level(self, mock_mddisplay):
|
||||
# should check if md level is valid
|
||||
# should raise error if it is not
|
||||
mock_mddisplay.return_value = [{'name': '/dev/md10'}]
|
||||
self.assertRaises(
|
||||
errors.MDWrongSpecError, mu.mdcreate,
|
||||
'/dev/md0', 'badlevel', '/dev/fake')
|
||||
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
def test_mdcreate_device_not_found(self, mock_mddisplay, mock_bdevs):
|
||||
# should check if all devices exist
|
||||
# should raise error if at least one device does not
|
||||
mock_mddisplay.return_value = [{'name': '/dev/md10'}]
|
||||
mock_bdevs.return_value = [{'device': '/dev/fake1'},
|
||||
{'device': '/dev/fake10'}]
|
||||
self.assertRaises(
|
||||
errors.MDNotFoundError, mu.mdcreate,
|
||||
'/dev/md0', 'mirror', '/dev/fake1', '/dev/fake2')
|
||||
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
def test_mdcreate_device_attached(self, mock_mddisplay, mock_bdevs):
|
||||
# should check if all necessary devices are not attached to some md
|
||||
# should raise error if at least one device is attached
|
||||
mock_mddisplay.return_value = [{'name': '/dev/md10',
|
||||
'devices': ['/dev/fake2']}]
|
||||
mock_bdevs.return_value = [{'device': '/dev/fake1'},
|
||||
{'device': '/dev/fake2'}]
|
||||
self.assertRaises(
|
||||
errors.MDDeviceDuplicationError, mu.mdcreate,
|
||||
'/dev/md0', 'mirror', '/dev/fake1', '/dev/fake2')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(mu, 'mdclean')
|
||||
@mock.patch.object(hu, 'list_block_devices')
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
def test_mdcreate_device_clean(self, mock_mddisplay,
|
||||
mock_bdevs, mock_mdclean, mock_exec):
|
||||
# should clean md metadata on all devices before building new md
|
||||
mock_mddisplay.return_value = []
|
||||
mock_bdevs.return_value = [{'device': '/dev/fake1'},
|
||||
{'device': '/dev/fake2'}]
|
||||
mu.mdcreate('/dev/md0', 'mirror', '/dev/fake1', '/dev/fake2')
|
||||
expected_calls = [mock.call('/dev/fake1'), mock.call('/dev/fake2')]
|
||||
self.assertEqual(mock_mdclean.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(mu, 'mdclean')
|
||||
@mock.patch.object(mu, 'mdremove')
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
def test_mdclean_all(self, mock_mddisplay, mock_mdremove, mock_mdclean):
|
||||
mock_mddisplay.side_effect = [
|
||||
[{'name': '/dev/md10', 'devices': ['/dev/fake10']},
|
||||
{'name': '/dev/md11'}],
|
||||
[{'name': '/dev/md11'}],
|
||||
[]
|
||||
]
|
||||
mu.mdclean_all()
|
||||
mock_mdremove_expected_calls = [
|
||||
mock.call('/dev/md10'), mock.call('/dev/md11'),
|
||||
mock.call('/dev/md11')]
|
||||
mock_mdclean.assert_called_once_with('/dev/fake10')
|
||||
self.assertEqual(mock_mdremove.call_args_list,
|
||||
mock_mdremove_expected_calls)
|
||||
|
||||
@mock.patch.object(mu, 'mdclean')
|
||||
@mock.patch.object(mu, 'mdremove')
|
||||
@mock.patch.object(mu, 'mddisplay')
|
||||
def test_mdclean_all_fail(self, mock_mddisplay, mock_mdremove,
|
||||
mock_mdclean):
|
||||
mock_mddisplay.return_value = [{'name': '/dev/md11'}]
|
||||
self.assertRaises(errors.MDRemovingError, mu.mdclean_all)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
@mock.patch.object(mu, 'get_mdnames')
|
||||
def test_mdremove_ok(self, mock_get_mdn, mock_exec):
|
||||
# should check if md exists
|
||||
# should run mdadm command to remove md device
|
||||
mock_get_mdn.return_value = ['/dev/md0']
|
||||
expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('mdadm', '--stop', '/dev/md0', check_exit_code=[0]),
|
||||
mock.call('mdadm', '--remove', '/dev/md0', check_exit_code=[0, 1])
|
||||
]
|
||||
mu.mdremove('/dev/md0')
|
||||
self.assertEqual(mock_exec.call_args_list, expected_calls)
|
||||
|
||||
@mock.patch.object(mu, 'get_mdnames')
|
||||
def test_mdremove_notfound(self, mock_get_mdn):
|
||||
# should check if md exists
|
||||
# should raise error if it does not
|
||||
mock_get_mdn.return_value = ['/dev/md0']
|
||||
self.assertRaises(
|
||||
errors.MDNotFoundError, mu.mdremove, '/dev/md1')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_mdclean(self, mock_exec):
|
||||
mu.mdclean('/dev/md0')
|
||||
mock_exec.assert_called_once_with('mdadm', '--zero-superblock',
|
||||
'--force', '/dev/md0',
|
||||
check_exit_code=[0])
|
File diff suppressed because it is too large
Load Diff
|
@ -1,244 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
import os
|
||||
import six
|
||||
from six.moves.urllib.parse import urlsplit
|
||||
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent.drivers.nailgun import NailgunBuildImage
|
||||
from fuel_agent import errors
|
||||
from fuel_agent import objects
|
||||
|
||||
DEFAULT_TRUSTY_PACKAGES = [
|
||||
"acl",
|
||||
"anacron",
|
||||
"bash-completion",
|
||||
"bridge-utils",
|
||||
"bsdmainutils",
|
||||
"build-essential",
|
||||
"cloud-init",
|
||||
"curl",
|
||||
"daemonize",
|
||||
"debconf-utils",
|
||||
"gdisk",
|
||||
"grub-pc",
|
||||
"linux-firmware",
|
||||
"linux-firmware-nonfree",
|
||||
"linux-headers-generic-lts-trusty",
|
||||
"linux-image-generic-lts-trusty",
|
||||
"lvm2",
|
||||
"mcollective",
|
||||
"mdadm",
|
||||
"nailgun-agent",
|
||||
"nailgun-mcagents",
|
||||
"nailgun-net-check",
|
||||
"ntp",
|
||||
"openssh-client",
|
||||
"openssh-server",
|
||||
"puppet",
|
||||
"python-amqp",
|
||||
"ruby-augeas",
|
||||
"ruby-ipaddress",
|
||||
"ruby-json",
|
||||
"ruby-netaddr",
|
||||
"ruby-openstack",
|
||||
"ruby-shadow",
|
||||
"ruby-stomp",
|
||||
"telnet",
|
||||
"ubuntu-minimal",
|
||||
"ubuntu-standard",
|
||||
"uuid-runtime",
|
||||
"vim",
|
||||
"virt-what",
|
||||
"vlan",
|
||||
]
|
||||
|
||||
REPOS_SAMPLE = [
|
||||
{
|
||||
"name": "ubuntu",
|
||||
"section": "main universe multiverse",
|
||||
"uri": "http://archive.ubuntu.com/ubuntu/",
|
||||
"priority": None,
|
||||
"suite": "trusty",
|
||||
"type": "deb"
|
||||
},
|
||||
{
|
||||
"name": "mos",
|
||||
"section": "main restricted",
|
||||
"uri": "http://10.20.0.2:8080/2014.2-6.1/ubuntu/x86_64",
|
||||
"priority": 1050,
|
||||
"suite": "mos6.1",
|
||||
"type": "deb"
|
||||
}
|
||||
]
|
||||
|
||||
IMAGE_DATA_SAMPLE = {
|
||||
"/boot": {
|
||||
"container": "gzip",
|
||||
"uri": "http://10.20.0.2:8080/path/to/img-boot.img.gz",
|
||||
"format": "ext2"
|
||||
},
|
||||
"/": {
|
||||
"container": "gzip",
|
||||
"uri": "http://10.20.0.2:8080/path/to/img.img.gz",
|
||||
"format": "ext4"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestNailgunBuildImage(test_base.BaseTestCase):
|
||||
|
||||
def test_default_trusty_packages(self):
|
||||
self.assertEqual(NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES,
|
||||
DEFAULT_TRUSTY_PACKAGES)
|
||||
|
||||
@mock.patch.object(NailgunBuildImage, '__init__')
|
||||
def test_parse_operating_system_error_bad_codename(self, mock_init):
|
||||
mock_init.return_value = None
|
||||
driver = NailgunBuildImage()
|
||||
driver.data = {'codename': 'not-trusty'}
|
||||
self.assertRaises(errors.WrongInputDataError,
|
||||
driver.parse_operating_system)
|
||||
|
||||
@mock.patch('fuel_agent.objects.Ubuntu')
|
||||
@mock.patch.object(NailgunBuildImage, '__init__')
|
||||
def test_parse_operating_system_packages_given(self, mock_init, mock_ub):
|
||||
mock_init.return_value = None
|
||||
data = {
|
||||
'repos': [],
|
||||
'codename': 'trusty',
|
||||
'packages': ['pack']
|
||||
}
|
||||
driver = NailgunBuildImage()
|
||||
driver.data = data
|
||||
mock_ub_instance = mock_ub.return_value
|
||||
mock_ub_instance.packages = data['packages']
|
||||
driver.parse_operating_system()
|
||||
mock_ub.assert_called_once_with(repos=[], packages=data['packages'])
|
||||
self.assertEqual(driver.operating_system.packages, data['packages'])
|
||||
|
||||
@mock.patch('fuel_agent.objects.Ubuntu')
|
||||
@mock.patch.object(NailgunBuildImage, '__init__')
|
||||
def test_parse_operating_system_packages_not_given(
|
||||
self, mock_init, mock_ub):
|
||||
mock_init.return_value = None
|
||||
data = {
|
||||
'repos': [],
|
||||
'codename': 'trusty'
|
||||
}
|
||||
driver = NailgunBuildImage()
|
||||
driver.data = data
|
||||
mock_ub_instance = mock_ub.return_value
|
||||
mock_ub_instance.packages = NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES
|
||||
driver.parse_operating_system()
|
||||
mock_ub.assert_called_once_with(
|
||||
repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
|
||||
self.assertEqual(driver.operating_system.packages,
|
||||
NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
|
||||
|
||||
@mock.patch('fuel_agent.objects.DEBRepo')
|
||||
@mock.patch('fuel_agent.objects.Ubuntu')
|
||||
@mock.patch.object(NailgunBuildImage, '__init__')
|
||||
def test_parse_operating_system_repos(self, mock_init, mock_ub, mock_deb):
|
||||
mock_init.return_value = None
|
||||
data = {
|
||||
'repos': REPOS_SAMPLE,
|
||||
'codename': 'trusty'
|
||||
}
|
||||
driver = NailgunBuildImage()
|
||||
driver.data = data
|
||||
|
||||
mock_deb_expected_calls = []
|
||||
repos = []
|
||||
for r in REPOS_SAMPLE:
|
||||
kwargs = {
|
||||
'name': r['name'],
|
||||
'uri': r['uri'],
|
||||
'suite': r['suite'],
|
||||
'section': r['section'],
|
||||
'priority': r['priority']
|
||||
}
|
||||
mock_deb_expected_calls.append(mock.call(**kwargs))
|
||||
repos.append(objects.DEBRepo(**kwargs))
|
||||
driver.parse_operating_system()
|
||||
mock_ub_instance = mock_ub.return_value
|
||||
mock_ub_instance.repos = repos
|
||||
mock_ub.assert_called_once_with(
|
||||
repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
|
||||
self.assertEqual(mock_deb_expected_calls,
|
||||
mock_deb.call_args_list[:len(REPOS_SAMPLE)])
|
||||
self.assertEqual(driver.operating_system.repos, repos)
|
||||
|
||||
@mock.patch('fuel_agent.drivers.nailgun.objects.Loop')
|
||||
@mock.patch('fuel_agent.objects.Image')
|
||||
@mock.patch('fuel_agent.objects.Fs')
|
||||
@mock.patch('fuel_agent.objects.PartitionScheme')
|
||||
@mock.patch('fuel_agent.objects.ImageScheme')
|
||||
@mock.patch.object(NailgunBuildImage, '__init__')
|
||||
def test_parse_schemes(
|
||||
self, mock_init, mock_imgsch, mock_partsch,
|
||||
mock_fs, mock_img, mock_loop):
|
||||
mock_init.return_value = None
|
||||
data = {
|
||||
'image_data': IMAGE_DATA_SAMPLE,
|
||||
'output': '/some/local/path',
|
||||
}
|
||||
driver = NailgunBuildImage()
|
||||
driver.data = data
|
||||
driver.parse_schemes()
|
||||
|
||||
mock_fs_expected_calls = []
|
||||
mock_img_expected_calls = []
|
||||
images = []
|
||||
fss = []
|
||||
data_length = len(data['image_data'].keys())
|
||||
for mount, image in six.iteritems(data['image_data']):
|
||||
filename = os.path.basename(urlsplit(image['uri']).path)
|
||||
img_kwargs = {
|
||||
'uri': 'file://' + os.path.join(data['output'], filename),
|
||||
'format': image['format'],
|
||||
'container': image['container'],
|
||||
'target_device': None
|
||||
}
|
||||
mock_img_expected_calls.append(mock.call(**img_kwargs))
|
||||
images.append(objects.Image(**img_kwargs))
|
||||
|
||||
fs_kwargs = {
|
||||
'device': None,
|
||||
'mount': mount,
|
||||
'fs_type': image['format']
|
||||
}
|
||||
mock_fs_expected_calls.append(mock.call(**fs_kwargs))
|
||||
fss.append(objects.Fs(**fs_kwargs))
|
||||
|
||||
if mount == '/':
|
||||
metadata_filename = filename.split('.', 1)[0] + '.yaml'
|
||||
|
||||
mock_imgsch_instance = mock_imgsch.return_value
|
||||
mock_imgsch_instance.images = images
|
||||
mock_partsch_instance = mock_partsch.return_value
|
||||
mock_partsch_instance.fss = fss
|
||||
|
||||
self.assertEqual(
|
||||
driver.metadata_uri, 'file://' + os.path.join(
|
||||
data['output'], metadata_filename))
|
||||
self.assertEqual(mock_img_expected_calls,
|
||||
mock_img.call_args_list[:data_length])
|
||||
self.assertEqual(mock_fs_expected_calls,
|
||||
mock_fs.call_args_list[:data_length])
|
||||
self.assertEqual(driver.image_scheme.images, images)
|
||||
self.assertEqual(driver.partition_scheme.fss, fss)
|
|
@ -1,256 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
|
||||
from oslotest import base as test_base
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.objects import partition
|
||||
|
||||
|
||||
class TestMD(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestMD, self).setUp()
|
||||
self.md = partition.Md('name', 'level')
|
||||
|
||||
def test_add_device_ok(self):
|
||||
self.assertEqual(0, len(self.md.devices))
|
||||
self.md.add_device('device')
|
||||
self.assertEqual(1, len(self.md.devices))
|
||||
self.assertEqual('device', self.md.devices[0])
|
||||
|
||||
def test_add_device_in_spares_fail(self):
|
||||
self.assertEqual(0, len(self.md.devices))
|
||||
self.assertEqual(0, len(self.md.spares))
|
||||
self.md.add_spare('device')
|
||||
self.assertRaises(errors.MDDeviceDuplicationError, self.md.add_device,
|
||||
'device')
|
||||
|
||||
def test_add_device_in_devices_fail(self):
|
||||
self.assertEqual(0, len(self.md.devices))
|
||||
self.assertEqual(0, len(self.md.spares))
|
||||
self.md.add_device('device')
|
||||
self.assertRaises(errors.MDDeviceDuplicationError, self.md.add_device,
|
||||
'device')
|
||||
|
||||
def test_add_spare_in_spares_fail(self):
|
||||
self.assertEqual(0, len(self.md.devices))
|
||||
self.assertEqual(0, len(self.md.spares))
|
||||
self.md.add_spare('device')
|
||||
self.assertRaises(errors.MDDeviceDuplicationError, self.md.add_spare,
|
||||
'device')
|
||||
|
||||
def test_add_spare_in_devices_fail(self):
|
||||
self.assertEqual(0, len(self.md.devices))
|
||||
self.assertEqual(0, len(self.md.spares))
|
||||
self.md.add_device('device')
|
||||
self.assertRaises(errors.MDDeviceDuplicationError, self.md.add_spare,
|
||||
'device')
|
||||
|
||||
|
||||
class TestPartition(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestPartition, self).setUp()
|
||||
self.pt = partition.Partition('name', 'count', 'device', 'begin',
|
||||
'end', 'partition_type')
|
||||
|
||||
def test_set_flag(self):
|
||||
self.assertEqual(0, len(self.pt.flags))
|
||||
self.pt.set_flag('fake_flag')
|
||||
self.assertEqual(1, len(self.pt.flags))
|
||||
self.assertIn('fake_flag', self.pt.flags)
|
||||
|
||||
|
||||
class TestPartitionScheme(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestPartitionScheme, self).setUp()
|
||||
self.p_scheme = partition.PartitionScheme()
|
||||
|
||||
def test_root_device_not_found(self):
|
||||
self.assertRaises(errors.WrongPartitionSchemeError,
|
||||
self.p_scheme.root_device)
|
||||
|
||||
def test_fs_by_device(self):
|
||||
expected_fs = partition.Fs('device')
|
||||
self.p_scheme.fss.append(expected_fs)
|
||||
self.p_scheme.fss.append(partition.Fs('wrong_device'))
|
||||
actual_fs = self.p_scheme.fs_by_device('device')
|
||||
self.assertEqual(expected_fs, actual_fs)
|
||||
|
||||
def test_fs_by_mount(self):
|
||||
expected_fs = partition.Fs('d', mount='mount')
|
||||
self.p_scheme.fss.append(expected_fs)
|
||||
self.p_scheme.fss.append(partition.Fs('w_d', mount='wrong_mount'))
|
||||
actual_fs = self.p_scheme.fs_by_mount('mount')
|
||||
self.assertEqual(expected_fs, actual_fs)
|
||||
|
||||
def test_pv_by_name(self):
|
||||
expected_pv = partition.Pv('pv')
|
||||
self.p_scheme.pvs.append(expected_pv)
|
||||
self.p_scheme.pvs.append(partition.Pv('wrong_pv'))
|
||||
actual_pv = self.p_scheme.pv_by_name('pv')
|
||||
self.assertEqual(expected_pv, actual_pv)
|
||||
|
||||
def test_vg_by_name(self):
|
||||
expected_vg = partition.Vg('vg')
|
||||
self.p_scheme.vgs.append(expected_vg)
|
||||
self.p_scheme.vgs.append(partition.Vg('wrong_vg'))
|
||||
actual_vg = self.p_scheme.vg_by_name('vg')
|
||||
self.assertEqual(expected_vg, actual_vg)
|
||||
|
||||
def test_vg_attach_by_name(self):
|
||||
self.p_scheme.vg_attach_by_name('pvname', 'vgname')
|
||||
self.assertEqual(1, len(self.p_scheme.pvs))
|
||||
self.assertEqual(1, len(self.p_scheme.vgs))
|
||||
self.assertIn('pvname', self.p_scheme.vgs[0].pvnames)
|
||||
self.assertIn('vgname', self.p_scheme.vgs[0].name)
|
||||
|
||||
def test_md_next_name_ok(self):
|
||||
expected_name = '/dev/md0'
|
||||
self.assertEqual(expected_name, self.p_scheme.md_next_name())
|
||||
|
||||
def test_md_next_name_fail(self):
|
||||
self.p_scheme.mds = [
|
||||
partition.Md('/dev/md%s' % x, 'level') for x in range(0, 128)]
|
||||
self.assertRaises(errors.MDAlreadyExistsError,
|
||||
self.p_scheme.md_next_name)
|
||||
|
||||
def test_md_by_name(self):
|
||||
self.assertEqual(0, len(self.p_scheme.mds))
|
||||
expected_md = partition.Md('name', 'level')
|
||||
self.p_scheme.mds.append(expected_md)
|
||||
self.p_scheme.mds.append(partition.Md('wrong_name', 'level'))
|
||||
self.assertEqual(expected_md, self.p_scheme.md_by_name('name'))
|
||||
|
||||
def test_md_by_mount(self):
|
||||
self.assertEqual(0, len(self.p_scheme.mds))
|
||||
self.assertEqual(0, len(self.p_scheme.fss))
|
||||
expected_md = partition.Md('name', 'level')
|
||||
expected_fs = partition.Fs('name', mount='mount')
|
||||
self.p_scheme.mds.append(expected_md)
|
||||
self.p_scheme.fss.append(expected_fs)
|
||||
self.p_scheme.fss.append(partition.Fs('wrong_name',
|
||||
mount='wrong_mount'))
|
||||
self.assertEqual(expected_md, self.p_scheme.md_by_mount('mount'))
|
||||
|
||||
def test_md_attach_by_mount_md_exists(self):
|
||||
self.assertEqual(0, len(self.p_scheme.mds))
|
||||
self.assertEqual(0, len(self.p_scheme.fss))
|
||||
expected_md = partition.Md('name', 'level')
|
||||
expected_fs = partition.Fs('name', mount='mount')
|
||||
self.p_scheme.mds.append(expected_md)
|
||||
self.p_scheme.fss.append(expected_fs)
|
||||
actual_md = self.p_scheme.md_attach_by_mount('device', 'mount')
|
||||
self.assertIn('device', actual_md.devices)
|
||||
self.assertEqual(expected_md, actual_md)
|
||||
|
||||
def test_md_attach_by_mount_no_md(self):
|
||||
self.assertEqual(0, len(self.p_scheme.mds))
|
||||
self.assertEqual(0, len(self.p_scheme.fss))
|
||||
actual_md = self.p_scheme.md_attach_by_mount(
|
||||
'device', 'mount', fs_type='fs_type', fs_options='-F',
|
||||
fs_label='fs_label', name='name', level='level')
|
||||
self.assertIn('device', actual_md.devices)
|
||||
self.assertEqual(1, len(self.p_scheme.fss))
|
||||
self.assertEqual('name', self.p_scheme.fss[0].device)
|
||||
self.assertEqual('mount', self.p_scheme.fss[0].mount)
|
||||
self.assertEqual('fs_type', self.p_scheme.fss[0].type)
|
||||
self.assertEqual('fs_label', self.p_scheme.fss[0].label)
|
||||
self.assertEqual('-F', self.p_scheme.fss[0].options)
|
||||
|
||||
|
||||
class TestParted(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestParted, self).setUp()
|
||||
self.prtd = partition.Parted('name', 'label')
|
||||
|
||||
@mock.patch.object(partition.Parted, 'next_count')
|
||||
@mock.patch.object(partition.Parted, 'next_type')
|
||||
def test_next_name_none(self, nt_mock, nc_mock):
|
||||
nc_mock.return_value = 1
|
||||
nt_mock.return_value = 'extended'
|
||||
self.assertEqual(None, self.prtd.next_name())
|
||||
|
||||
@mock.patch.object(partition.Parted, 'next_count')
|
||||
@mock.patch.object(partition.Parted, 'next_type')
|
||||
def test_next_name_no_separator(self, nt_mock, nc_mock):
|
||||
nc_mock.return_value = 1
|
||||
nt_mock.return_value = 'not_extended'
|
||||
expected_name = '%s%s' % (self.prtd.name, 1)
|
||||
self.assertEqual(expected_name, self.prtd.next_name())
|
||||
|
||||
@mock.patch.object(partition.Parted, 'next_count')
|
||||
@mock.patch.object(partition.Parted, 'next_type')
|
||||
def test_next_name_with_separator(self, nt_mock, nc_mock):
|
||||
nc_mock.return_value = 1
|
||||
nt_mock.return_value = 'not_extended'
|
||||
self.prtd.name = '/dev/cciss/c0d0'
|
||||
expected_name = '%sp%s' % (self.prtd.name, 1)
|
||||
self.assertEqual(expected_name, self.prtd.next_name())
|
||||
self.prtd.name = '/dev/loop123'
|
||||
expected_name = '%sp%s' % (self.prtd.name, 1)
|
||||
self.assertEqual(expected_name, self.prtd.next_name())
|
||||
self.prtd.name = '/dev/nvme0n1'
|
||||
expected_name = '%sp%s' % (self.prtd.name, 1)
|
||||
self.assertEqual(expected_name, self.prtd.next_name())
|
||||
|
||||
def test_next_begin_empty_partitions(self):
|
||||
self.assertEqual(1, self.prtd.next_begin())
|
||||
|
||||
def test_next_begin_last_extended_partition(self):
|
||||
self.prtd.partitions.append(
|
||||
partition.Partition('name', 'count', 'device', 'begin', 'end',
|
||||
'extended'))
|
||||
self.assertEqual('begin', self.prtd.next_begin())
|
||||
|
||||
def test_next_begin_no_last_extended_partition(self):
|
||||
self.prtd.partitions.append(
|
||||
partition.Partition('name', 'count', 'device', 'begin', 'end',
|
||||
'primary'))
|
||||
self.assertEqual('end', self.prtd.next_begin())
|
||||
|
||||
def test_next_count_no_logical(self):
|
||||
self.assertEqual(1, self.prtd.next_count('primary'))
|
||||
|
||||
def test_next_count_has_logical(self):
|
||||
self.prtd.partitions.append(
|
||||
partition.Partition('name', 'count', 'device', 'begin', 'end',
|
||||
'logical'))
|
||||
self.assertEqual(6, self.prtd.next_count('logical'))
|
||||
|
||||
def test_next_type_gpt(self):
|
||||
self.prtd.label = 'gpt'
|
||||
self.assertEqual('primary', self.prtd.next_type())
|
||||
|
||||
def test_next_type_no_extended(self):
|
||||
self.prtd.label = 'msdos'
|
||||
self.assertEqual('primary', self.prtd.next_type())
|
||||
self.prtd.partitions.extend(
|
||||
3 * [partition.Partition('name', 'count', 'device', 'begin',
|
||||
'end', 'primary')])
|
||||
self.assertEqual('extended', self.prtd.next_type())
|
||||
|
||||
def test_next_type_has_extended(self):
|
||||
self.prtd.label = 'msdos'
|
||||
self.prtd.partitions.append(
|
||||
partition.Partition('name', 'count', 'device', 'begin', 'end',
|
||||
'extended'))
|
||||
self.assertEqual('logical', self.prtd.next_type())
|
||||
|
||||
def test_primary(self):
|
||||
expected_partitions = [partition.Partition('name', 'count', 'device',
|
||||
'begin', 'end', 'primary')]
|
||||
self.prtd.partitions.extend(expected_partitions)
|
||||
self.assertEqual(expected_partitions, self.prtd.primary)
|
|
@ -1,289 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslotest import base as test_base
|
||||
import time
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import partition as pu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
class TestPartitionUtils(test_base.BaseTestCase):
|
||||
@mock.patch.object(pu, 'make_label')
|
||||
def test_wipe(self, mock_label):
|
||||
# should run call make_label method
|
||||
# in order to create new empty table which we think
|
||||
# is equivalent to wiping the old one
|
||||
pu.wipe('/dev/fake')
|
||||
mock_label.assert_called_once_with('/dev/fake')
|
||||
|
||||
@mock.patch.object(pu, 'reread_partitions')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_make_label(self, mock_exec, mock_rerd):
|
||||
# should run parted OS command
|
||||
# in order to create label on a device
|
||||
mock_exec.return_value = ('out', '')
|
||||
|
||||
# gpt by default
|
||||
pu.make_label('/dev/fake')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('parted', '-s', '/dev/fake', 'mklabel', 'gpt',
|
||||
check_exit_code=[0, 1])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
mock_rerd.assert_called_once_with('/dev/fake', out='out')
|
||||
mock_exec.reset_mock()
|
||||
mock_rerd.reset_mock()
|
||||
|
||||
# label is set explicitly
|
||||
pu.make_label('/dev/fake', label='msdos')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('parted', '-s', '/dev/fake', 'mklabel', 'msdos',
|
||||
check_exit_code=[0, 1])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
mock_rerd.assert_called_once_with('/dev/fake', out='out')
|
||||
|
||||
def test_make_label_wrong_label(self):
|
||||
# should check if label is valid
|
||||
# should raise exception if it is not
|
||||
self.assertRaises(errors.WrongPartitionLabelError,
|
||||
pu.make_label, '/dev/fake', 'wrong')
|
||||
|
||||
@mock.patch.object(pu, 'reread_partitions')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_set_partition_flag(self, mock_exec, mock_rerd):
|
||||
# should run parted OS command
|
||||
# in order to set flag on a partition
|
||||
mock_exec.return_value = ('out', '')
|
||||
|
||||
# default state is 'on'
|
||||
pu.set_partition_flag('/dev/fake', 1, 'boot')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('parted', '-s', '/dev/fake', 'set', '1', 'boot', 'on',
|
||||
check_exit_code=[0, 1])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
mock_rerd.assert_called_once_with('/dev/fake', out='out')
|
||||
mock_exec.reset_mock()
|
||||
mock_rerd.reset_mock()
|
||||
|
||||
# if state argument is given use it
|
||||
pu.set_partition_flag('/dev/fake', 1, 'boot', state='off')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('parted', '-s', '/dev/fake', 'set', '1', 'boot', 'off',
|
||||
check_exit_code=[0, 1])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
mock_rerd.assert_called_once_with('/dev/fake', out='out')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_set_partition_flag_wrong_flag(self, mock_exec):
|
||||
# should check if flag is valid
|
||||
# should raise exception if it is not
|
||||
self.assertRaises(errors.WrongPartitionSchemeError,
|
||||
pu.set_partition_flag,
|
||||
'/dev/fake', 1, 'wrong')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_set_partition_flag_wrong_state(self, mock_exec):
|
||||
# should check if flag is valid
|
||||
# should raise exception if it is not
|
||||
self.assertRaises(errors.WrongPartitionSchemeError,
|
||||
pu.set_partition_flag,
|
||||
'/dev/fake', 1, 'boot', state='wrong')
|
||||
|
||||
@mock.patch.object(pu, 'reread_partitions')
|
||||
@mock.patch.object(pu, 'info')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_make_partition(self, mock_exec, mock_info, mock_rerd):
|
||||
# should run parted OS command
|
||||
# in order to create new partition
|
||||
mock_exec.return_value = ('out', '')
|
||||
|
||||
mock_info.return_value = {
|
||||
'parts': [
|
||||
{'begin': 0, 'end': 1000, 'fstype': 'free'},
|
||||
]
|
||||
}
|
||||
pu.make_partition('/dev/fake', 100, 200, 'primary')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('parted', '-a', 'optimal', '-s', '/dev/fake', 'unit',
|
||||
'MiB', 'mkpart', 'primary', '100', '200',
|
||||
check_exit_code=[0, 1])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
mock_rerd.assert_called_once_with('/dev/fake', out='out')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_make_partition_wrong_ptype(self, mock_exec):
|
||||
# should check if partition type is one of
|
||||
# 'primary' or 'logical'
|
||||
# should raise exception if it is not
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, pu.make_partition,
|
||||
'/dev/fake', 200, 100, 'wrong')
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_make_partition_begin_overlaps_end(self, mock_exec):
|
||||
# should check if begin is less than end
|
||||
# should raise exception if it isn't
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, pu.make_partition,
|
||||
'/dev/fake', 200, 100, 'primary')
|
||||
|
||||
@mock.patch.object(pu, 'info')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_make_partition_overlaps_other_parts(self, mock_exec, mock_info):
|
||||
# should check if begin or end overlap other partitions
|
||||
# should raise exception if it does
|
||||
mock_info.return_value = {
|
||||
'parts': [
|
||||
{'begin': 0, 'end': 100, 'fstype': 'free'},
|
||||
{'begin': 100, 'end': 200, 'fstype': 'notfree'},
|
||||
{'begin': 200, 'end': 300, 'fstype': 'free'}
|
||||
]
|
||||
}
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, pu.make_partition,
|
||||
'/dev/fake', 99, 101, 'primary')
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, pu.make_partition,
|
||||
'/dev/fake', 100, 200, 'primary')
|
||||
self.assertRaises(errors.WrongPartitionSchemeError, pu.make_partition,
|
||||
'/dev/fake', 200, 301, 'primary')
|
||||
self.assertEqual(mock_info.call_args_list,
|
||||
[mock.call('/dev/fake')] * 3)
|
||||
|
||||
@mock.patch.object(pu, 'reread_partitions')
|
||||
@mock.patch.object(pu, 'info')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_remove_partition(self, mock_exec, mock_info, mock_rerd):
|
||||
# should run parted OS command
|
||||
# in order to remove partition
|
||||
mock_exec.return_value = ('out', '')
|
||||
mock_info.return_value = {
|
||||
'parts': [
|
||||
{
|
||||
'begin': 1,
|
||||
'end': 100,
|
||||
'size': 100,
|
||||
'num': 1,
|
||||
'fstype': 'ext2'
|
||||
},
|
||||
{
|
||||
'begin': 100,
|
||||
'end': 200,
|
||||
'size': 100,
|
||||
'num': 2,
|
||||
'fstype': 'ext2'
|
||||
}
|
||||
]
|
||||
}
|
||||
pu.remove_partition('/dev/fake', 1)
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('parted', '-s', '/dev/fake', 'rm', '1',
|
||||
check_exit_code=[0, 1])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
mock_rerd.assert_called_once_with('/dev/fake', out='out')
|
||||
|
||||
@mock.patch.object(pu, 'info')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_remove_partition_notexists(self, mock_exec, mock_info):
|
||||
# should check if partition does exist
|
||||
# should raise exception if it doesn't
|
||||
mock_info.return_value = {
|
||||
'parts': [
|
||||
{
|
||||
'begin': 1,
|
||||
'end': 100,
|
||||
'size': 100,
|
||||
'num': 1,
|
||||
'fstype': 'ext2'
|
||||
},
|
||||
{
|
||||
'begin': 100,
|
||||
'end': 200,
|
||||
'size': 100,
|
||||
'num': 2,
|
||||
'fstype': 'ext2'
|
||||
}
|
||||
]
|
||||
}
|
||||
self.assertRaises(errors.PartitionNotFoundError, pu.remove_partition,
|
||||
'/dev/fake', 3)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_set_gpt_type(self, mock_exec):
|
||||
pu.set_gpt_type('dev', 'num', 'type')
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('sgdisk', '--typecode=%s:%s' % ('num', 'type'), 'dev',
|
||||
check_exit_code=[0])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_info(self, mock_exec):
|
||||
mock_exec.return_value = [
|
||||
'BYT;\n'
|
||||
'/dev/fake:476940MiB:scsi:512:4096:msdos:ATA 1BD14;\n'
|
||||
'1:0.03MiB:1.00MiB:0.97MiB:free;\n'
|
||||
'1:1.00MiB:191MiB:190MiB:ext3::boot;\n'
|
||||
'2:191MiB:476939MiB:476748MiB:::lvm;\n'
|
||||
'1:476939MiB:476940MiB:1.02MiB:free;\n'
|
||||
]
|
||||
expected = {'generic': {'dev': '/dev/fake',
|
||||
'logical_block': 512,
|
||||
'model': 'ATA 1BD14',
|
||||
'physical_block': 4096,
|
||||
'size': 476940,
|
||||
'table': 'msdos'},
|
||||
|
||||
'parts': [{'begin': 1, 'end': 1, 'fstype': 'free',
|
||||
'num': 1, 'size': 1},
|
||||
{'begin': 1, 'end': 191, 'fstype': 'ext3',
|
||||
'num': 1, 'size': 190},
|
||||
{'begin': 191, 'end': 476939, 'fstype': None,
|
||||
'num': 2, 'size': 476748},
|
||||
{'begin': 476939, 'end': 476940,
|
||||
'fstype': 'free', 'num': 1, 'size': 2}]}
|
||||
actual = pu.info('/dev/fake')
|
||||
self.assertEqual(expected, actual)
|
||||
mock_exec_expected_calls = [
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
mock.call('parted', '-s', '/dev/fake', '-m', 'unit', 'MiB',
|
||||
'print', 'free', check_exit_code=[0])]
|
||||
self.assertEqual(mock_exec_expected_calls, mock_exec.call_args_list)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_reread_partitions_ok(self, mock_exec):
|
||||
pu.reread_partitions('/dev/fake', out='')
|
||||
self.assertEqual(mock_exec.call_args_list, [])
|
||||
|
||||
@mock.patch.object(time, 'sleep')
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_reread_partitions_device_busy(self, mock_exec, mock_sleep):
|
||||
mock_exec.return_value = ('', '')
|
||||
pu.reread_partitions('/dev/fake', out='_Device or resource busy_')
|
||||
mock_exec_expected = [
|
||||
mock.call('partprobe', '/dev/fake', check_exit_code=[0, 1]),
|
||||
mock.call('udevadm', 'settle', '--quiet', check_exit_code=[0]),
|
||||
]
|
||||
self.assertEqual(mock_exec.call_args_list, mock_exec_expected)
|
||||
mock_sleep.assert_called_once_with(2)
|
||||
|
||||
@mock.patch.object(utils, 'execute')
|
||||
def test_reread_partitions_timeout(self, mock_exec):
|
||||
self.assertRaises(errors.BaseError, pu.reread_partitions,
|
||||
'/dev/fake', out='Device or resource busy',
|
||||
timeout=-40)
|
|
@ -1,214 +0,0 @@
|
|||
# Copyright 2011 Justin Santa Barbara
|
||||
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import testtools
|
||||
|
||||
import mock
|
||||
from oslo.config import cfg
|
||||
import requests
|
||||
import socket
|
||||
import stevedore
|
||||
import urllib3
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class ExecuteTestCase(testtools.TestCase):
|
||||
"""This class is partly based on the same class in openstack/ironic."""
|
||||
|
||||
def setUp(self):
|
||||
super(ExecuteTestCase, self).setUp()
|
||||
fake_driver = stevedore.extension.Extension('fake_driver', None, None,
|
||||
mock.MagicMock)
|
||||
self.drv_manager = stevedore.driver.DriverManager.make_test_instance(
|
||||
fake_driver)
|
||||
|
||||
def test_parse_unit(self):
|
||||
self.assertEqual(utils.parse_unit('1.00m', 'm', ceil=True), 1)
|
||||
self.assertEqual(utils.parse_unit('1.00m', 'm', ceil=False), 1)
|
||||
self.assertEqual(utils.parse_unit('1.49m', 'm', ceil=True), 2)
|
||||
self.assertEqual(utils.parse_unit('1.49m', 'm', ceil=False), 1)
|
||||
self.assertEqual(utils.parse_unit('1.51m', 'm', ceil=True), 2)
|
||||
self.assertEqual(utils.parse_unit('1.51m', 'm', ceil=False), 1)
|
||||
self.assertRaises(ValueError, utils.parse_unit, '1.00m', 'MiB')
|
||||
self.assertRaises(ValueError, utils.parse_unit, '', 'MiB')
|
||||
|
||||
def test_B2MiB(self):
|
||||
self.assertEqual(utils.B2MiB(1048575, ceil=False), 0)
|
||||
self.assertEqual(utils.B2MiB(1048576, ceil=False), 1)
|
||||
self.assertEqual(utils.B2MiB(1048575, ceil=True), 1)
|
||||
self.assertEqual(utils.B2MiB(1048576, ceil=True), 1)
|
||||
self.assertEqual(utils.B2MiB(1048577, ceil=True), 2)
|
||||
|
||||
def test_check_exit_code_boolean(self):
|
||||
utils.execute('/usr/bin/env', 'false', check_exit_code=False)
|
||||
self.assertRaises(errors.ProcessExecutionError,
|
||||
utils.execute,
|
||||
'/usr/bin/env', 'false', check_exit_code=True)
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.time.sleep')
|
||||
@mock.patch('fuel_agent.utils.utils.subprocess.Popen')
|
||||
def test_execute_ok_on_third_attempts(self, mock_popen, mock_sleep):
|
||||
process = mock.Mock()
|
||||
mock_popen.side_effect = [OSError, ValueError, process]
|
||||
process.communicate.return_value = (None, None)
|
||||
process.returncode = 0
|
||||
utils.execute('/usr/bin/env', 'false', attempts=3)
|
||||
self.assertEqual(2 * [mock.call(CONF.execute_retry_delay)],
|
||||
mock_sleep.call_args_list)
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.time.sleep')
|
||||
@mock.patch('fuel_agent.utils.utils.subprocess.Popen')
|
||||
def test_execute_failed(self, mock_popen, mock_sleep):
|
||||
mock_popen.side_effect = OSError
|
||||
self.assertRaises(errors.ProcessExecutionError, utils.execute,
|
||||
'/usr/bin/env', 'false', attempts=2)
|
||||
self.assertEqual(1 * [mock.call(CONF.execute_retry_delay)],
|
||||
mock_sleep.call_args_list)
|
||||
|
||||
@mock.patch('stevedore.driver.DriverManager')
|
||||
def test_get_driver(self, mock_drv_manager):
|
||||
mock_drv_manager.return_value = self.drv_manager
|
||||
self.assertEqual(mock.MagicMock.__name__,
|
||||
utils.get_driver('fake_driver').__name__)
|
||||
|
||||
@mock.patch('jinja2.Environment')
|
||||
@mock.patch('jinja2.FileSystemLoader')
|
||||
@mock.patch('six.moves.builtins.open')
|
||||
def test_render_and_save_fail(self, mock_open, mock_j_lo, mock_j_env):
|
||||
mock_open.side_effect = Exception('foo')
|
||||
self.assertRaises(errors.TemplateWriteError, utils.render_and_save,
|
||||
'fake_dir', 'fake_tmpl_name', 'fake_data',
|
||||
'fake_file_name')
|
||||
|
||||
@mock.patch('jinja2.Environment')
|
||||
@mock.patch('jinja2.FileSystemLoader')
|
||||
@mock.patch('six.moves.builtins.open')
|
||||
def test_render_and_save_ok(self, mock_open, mock_j_lo, mock_j_env):
|
||||
mock_render = mock.Mock()
|
||||
mock_render.render.return_value = 'fake_data'
|
||||
mock_j_env.get_template.return_value = mock_render
|
||||
utils.render_and_save('fake_dir', 'fake_tmpl_name', 'fake_data',
|
||||
'fake_file_name')
|
||||
mock_open.assert_called_once_with('fake_file_name', 'w')
|
||||
|
||||
def test_calculate_md5_ok(self):
|
||||
# calculated by 'printf %10000s | md5sum'
|
||||
with mock.patch('six.moves.builtins.open',
|
||||
mock.mock_open(read_data=' ' * 10000), create=True):
|
||||
self.assertEqual('f38898bb69bb02bccb9594dfe471c5c0',
|
||||
utils.calculate_md5('fake', 10000))
|
||||
self.assertEqual('6934d9d33cd2d0c005994e7d96d2e0d9',
|
||||
utils.calculate_md5('fake', 1000))
|
||||
self.assertEqual('1e68934346ee57858834a205017af8b7',
|
||||
utils.calculate_md5('fake', 100))
|
||||
self.assertEqual('41b394758330c83757856aa482c79977',
|
||||
utils.calculate_md5('fake', 10))
|
||||
self.assertEqual('7215ee9c7d9dc229d2921a40e899ec5f',
|
||||
utils.calculate_md5('fake', 1))
|
||||
self.assertEqual('d41d8cd98f00b204e9800998ecf8427e',
|
||||
utils.calculate_md5('fake', 0))
|
||||
|
||||
@mock.patch.object(requests, 'get')
|
||||
def test_init_http_request_ok(self, mock_req):
|
||||
utils.init_http_request('fake_url')
|
||||
mock_req.assert_called_once_with(
|
||||
'fake_url', stream=True, timeout=CONF.http_request_timeout,
|
||||
headers={'Range': 'bytes=0-'})
|
||||
|
||||
@mock.patch('time.sleep')
|
||||
@mock.patch.object(requests, 'get')
|
||||
def test_init_http_request_non_critical_errors(self, mock_req, mock_s):
|
||||
mock_ok = mock.Mock()
|
||||
mock_req.side_effect = [urllib3.exceptions.DecodeError(),
|
||||
urllib3.exceptions.ProxyError(),
|
||||
requests.exceptions.ConnectionError(),
|
||||
requests.exceptions.Timeout(),
|
||||
requests.exceptions.TooManyRedirects(),
|
||||
socket.timeout(),
|
||||
mock_ok]
|
||||
req_obj = utils.init_http_request('fake_url')
|
||||
self.assertEqual(mock_ok, req_obj)
|
||||
|
||||
@mock.patch.object(requests, 'get')
|
||||
def test_init_http_request_wrong_http_status(self, mock_req):
|
||||
mock_fail = mock.Mock()
|
||||
mock_fail.raise_for_status.side_effect = KeyError()
|
||||
mock_req.return_value = mock_fail
|
||||
self.assertRaises(KeyError, utils.init_http_request, 'fake_url')
|
||||
|
||||
@mock.patch('time.sleep')
|
||||
@mock.patch.object(requests, 'get')
|
||||
def test_init_http_request_max_retries_exceeded(self, mock_req, mock_s):
|
||||
mock_req.side_effect = requests.exceptions.ConnectionError()
|
||||
self.assertRaises(errors.HttpUrlConnectionError,
|
||||
utils.init_http_request, 'fake_url')
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.os.makedirs')
|
||||
@mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=False)
|
||||
def test_makedirs_if_not_exists(self, mock_isdir, mock_makedirs):
|
||||
utils.makedirs_if_not_exists('/fake/path')
|
||||
mock_isdir.assert_called_once_with('/fake/path')
|
||||
mock_makedirs.assert_called_once_with('/fake/path', mode=0o755)
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.os.makedirs')
|
||||
@mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=False)
|
||||
def test_makedirs_if_not_exists_mode_given(
|
||||
self, mock_isdir, mock_makedirs):
|
||||
utils.makedirs_if_not_exists('/fake/path', mode=0o000)
|
||||
mock_isdir.assert_called_once_with('/fake/path')
|
||||
mock_makedirs.assert_called_once_with('/fake/path', mode=0o000)
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.os.makedirs')
|
||||
@mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=True)
|
||||
def test_makedirs_if_not_exists_already_exists(
|
||||
self, mock_isdir, mock_makedirs):
|
||||
utils.makedirs_if_not_exists('/fake/path')
|
||||
mock_isdir.assert_called_once_with('/fake/path')
|
||||
self.assertEqual(mock_makedirs.mock_calls, [])
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.os.listdir')
|
||||
def test_guess_filename(self, mock_oslistdir):
|
||||
mock_oslistdir.return_value = ['file1', 'file2', 'file3']
|
||||
filename = utils.guess_filename('/some/path', '^file2.*')
|
||||
self.assertEqual(filename, 'file2')
|
||||
mock_oslistdir.assert_called_once_with('/some/path')
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.os.listdir')
|
||||
def test_guess_filename_not_found(self, mock_oslistdir):
|
||||
mock_oslistdir.return_value = ['file1', 'file2', 'file3']
|
||||
filename = utils.guess_filename('/some/path', '^file4.*')
|
||||
self.assertIsNone(filename)
|
||||
mock_oslistdir.assert_called_once_with('/some/path')
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.os.listdir')
|
||||
def test_guess_filename_not_exact_match(self, mock_oslistdir):
|
||||
mock_oslistdir.return_value = ['file1', 'file2', 'file3']
|
||||
filename = utils.guess_filename('/some/path', '^file.*')
|
||||
# by default files are sorted in backward direction
|
||||
self.assertEqual(filename, 'file3')
|
||||
mock_oslistdir.assert_called_once_with('/some/path')
|
||||
|
||||
@mock.patch('fuel_agent.utils.utils.os.listdir')
|
||||
def test_guess_filename_not_exact_match_forward_sort(self, mock_oslistdir):
|
||||
mock_oslistdir.return_value = ['file1', 'file2', 'file3']
|
||||
filename = utils.guess_filename('/some/path', '^file.*', reverse=False)
|
||||
# by default files are sorted in backward direction
|
||||
self.assertEqual(filename, 'file1')
|
||||
mock_oslistdir.assert_called_once_with('/some/path')
|
|
@ -1,226 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
import os
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zlib
|
||||
|
||||
from oslo.config import cfg
|
||||
import six
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
au_opts = [
|
||||
cfg.IntOpt(
|
||||
'data_chunk_size',
|
||||
default=1048576,
|
||||
help='Size of data chunk to operate with images'
|
||||
),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(au_opts)
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class Target(object):
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
raise StopIteration()
|
||||
|
||||
def target(self, filename='/dev/null'):
|
||||
LOG.debug('Opening file: %s for write' % filename)
|
||||
with open(filename, 'wb') as f:
|
||||
count = 0
|
||||
for chunk in self:
|
||||
LOG.debug('Next chunk: %s' % count)
|
||||
f.write(chunk)
|
||||
count += 1
|
||||
LOG.debug('Flushing file: %s' % filename)
|
||||
f.flush()
|
||||
# ensure data to be written to disk
|
||||
os.fsync(f.fileno())
|
||||
LOG.debug('File is written: %s' % filename)
|
||||
|
||||
|
||||
class LocalFile(Target):
|
||||
def __init__(self, filename):
|
||||
if filename.startswith('file://'):
|
||||
self.filename = filename[7:]
|
||||
else:
|
||||
self.filename = filename
|
||||
self.fileobj = None
|
||||
|
||||
def next(self):
|
||||
if not self.fileobj:
|
||||
self.fileobj = open(self.filename, 'rb')
|
||||
buffer = self.fileobj.read(CONF.data_chunk_size)
|
||||
if buffer:
|
||||
return buffer
|
||||
else:
|
||||
self.fileobj.close()
|
||||
raise StopIteration()
|
||||
|
||||
|
||||
class HttpUrl(Target):
|
||||
def __init__(self, url):
|
||||
self.url = str(url)
|
||||
self.response_obj = utils.init_http_request(self.url)
|
||||
self.processed_bytes = 0
|
||||
try:
|
||||
self.length = int(self.response_obj.headers['content-length'])
|
||||
except (ValueError, KeyError):
|
||||
raise errors.HttpUrlInvalidContentLength(
|
||||
'Can not get content length for %s' % self.url)
|
||||
else:
|
||||
LOG.debug('Expected content length %s for %s' % (self.length,
|
||||
self.url))
|
||||
|
||||
def next(self):
|
||||
while self.processed_bytes < self.length:
|
||||
try:
|
||||
data = self.response_obj.raw.read(CONF.data_chunk_size)
|
||||
if not data:
|
||||
raise errors.HttpUrlConnectionError(
|
||||
'Could not receive data: URL=%s, range=%s' %
|
||||
(self.url, self.processed_bytes))
|
||||
except Exception as exc:
|
||||
LOG.exception(exc)
|
||||
self.response_obj = utils.init_http_request(
|
||||
self.url, self.processed_bytes)
|
||||
continue
|
||||
else:
|
||||
self.processed_bytes += len(data)
|
||||
return data
|
||||
raise StopIteration()
|
||||
|
||||
|
||||
class GunzipStream(Target):
|
||||
def __init__(self, stream):
|
||||
self.stream = iter(stream)
|
||||
# NOTE(agordeev): toggle automatic header detection on
|
||||
self.decompressor = zlib.decompressobj(zlib.MAX_WBITS | 32)
|
||||
|
||||
def next(self):
|
||||
try:
|
||||
return self.decompressor.decompress(self.stream.next())
|
||||
except StopIteration:
|
||||
raise
|
||||
|
||||
|
||||
class ForwardFileStream(Target):
|
||||
def __init__(self, stream):
|
||||
self.stream = iter(stream)
|
||||
self.position = 0
|
||||
self.chunk = ''
|
||||
self.closed = False
|
||||
|
||||
def next(self):
|
||||
buffer = self.read()
|
||||
if buffer:
|
||||
return buffer
|
||||
else:
|
||||
raise StopIteration()
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
def tell(self):
|
||||
if self.closed:
|
||||
raise ValueError('I/O operation on closed file')
|
||||
return self.position
|
||||
|
||||
def seek(self, position):
|
||||
if self.closed:
|
||||
raise ValueError('I/O operation on closed file')
|
||||
|
||||
if position < self.position:
|
||||
raise ValueError('Backward seek operation is impossible')
|
||||
elif position < self.position + len(self.chunk):
|
||||
self.chunk = self.chunk[(position - self.position):]
|
||||
self.position = position
|
||||
else:
|
||||
try:
|
||||
current = self.position + len(self.chunk)
|
||||
while True:
|
||||
chunk = self.stream.next()
|
||||
if current + len(chunk) >= position:
|
||||
self.chunk = chunk[(position - current):]
|
||||
self.position = position
|
||||
break
|
||||
current += len(chunk)
|
||||
except StopIteration:
|
||||
self.chunk = None
|
||||
self.position = position
|
||||
|
||||
def read(self, length=CONF.data_chunk_size):
|
||||
# NOTE(kozhukalov): default lenght = 1048576 is not usual behaviour,
|
||||
# but that is ok for our use case.
|
||||
if self.closed:
|
||||
raise ValueError('I/O operation on closed file')
|
||||
if self.chunk is None:
|
||||
return None
|
||||
try:
|
||||
while len(self.chunk) < length:
|
||||
self.chunk += self.stream.next()
|
||||
result = self.chunk[:length]
|
||||
self.chunk = self.chunk[length:]
|
||||
except StopIteration:
|
||||
result = self.chunk
|
||||
self.chunk = None
|
||||
self.position += len(result)
|
||||
return result
|
||||
|
||||
|
||||
class TarStream(Target):
|
||||
def __init__(self, stream):
|
||||
self.stream = iter(stream)
|
||||
self.tarobj = None
|
||||
|
||||
def target(self, filename=None):
|
||||
if not self.tarobj:
|
||||
self.tarobj = tarfile.open(
|
||||
fileobj=ForwardFileStream(self.stream), mode='r:')
|
||||
self.tarobj.extractall(path=(filename or tempfile.gettempdir()))
|
||||
|
||||
|
||||
class Chain(object):
|
||||
def __init__(self):
|
||||
self.processors = []
|
||||
|
||||
def append(self, processor):
|
||||
self.processors.append(processor)
|
||||
|
||||
def process(self):
|
||||
def jump(proc, next_proc):
|
||||
# if next_proc is just a string we assume it is a filename
|
||||
# and we save stream into a file
|
||||
if isinstance(next_proc, (str, unicode)):
|
||||
LOG.debug('Processor target: %s' % next_proc)
|
||||
proc.target(next_proc)
|
||||
return LocalFile(next_proc)
|
||||
# if next_proc is not a string we return new instance
|
||||
# initialized with the previous one
|
||||
else:
|
||||
return next_proc(proc)
|
||||
return reduce(jump, self.processors)
|
|
@ -1,502 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import gzip
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal as sig
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import hardware as hu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
bu_opts = [
|
||||
cfg.IntOpt(
|
||||
'max_loop_devices_count',
|
||||
default=255,
|
||||
# NOTE(agordeev): up to 256 loop devices could be allocated up to
|
||||
# kernel version 2.6.23, and the limit (from version 2.6.24 onwards)
|
||||
# isn't theoretically present anymore.
|
||||
help='Maximum allowed loop devices count to use'
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'sparse_file_size',
|
||||
# XXX: Apparently Fuel configures the node root filesystem to span
|
||||
# the whole hard drive. However 2 GB filesystem created with default
|
||||
# options can grow at most to 2 TB (1024x its initial size). This
|
||||
# maximal size can be configured by mke2fs -E resize=NNN option,
|
||||
# however the version of e2fsprogs shipped with CentOS 6.[65] seems
|
||||
# to silently ignore the `resize' option. Therefore make the initial
|
||||
# filesystem a bit bigger so it can grow to 8 TB.
|
||||
default=8192,
|
||||
help='Size of sparse file in MiBs'
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'loop_device_major_number',
|
||||
default=7,
|
||||
help='System-wide major number for loop device'
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'fetch_packages_attempts',
|
||||
default=10,
|
||||
help='Maximum allowed debootstrap/apt-get attempts to execute'
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'allow_unsigned_file',
|
||||
default='allow_unsigned_packages',
|
||||
help='File where to store apt setting for unsigned packages'
|
||||
),
|
||||
cfg.StrOpt(
|
||||
'force_ipv4_file',
|
||||
default='force_ipv4',
|
||||
help='File where to store apt setting for forcing IPv4 usage'
|
||||
),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(bu_opts)
|
||||
DEFAULT_APT_PATH = {
|
||||
'sources_file': 'etc/apt/sources.list',
|
||||
'sources_dir': 'etc/apt/sources.list.d',
|
||||
'preferences_file': 'etc/apt/preferences',
|
||||
'preferences_dir': 'etc/apt/preferences.d',
|
||||
'conf_dir': 'etc/apt/apt.conf.d',
|
||||
}
|
||||
# NOTE(agordeev): hardcoded to r00tme
|
||||
ROOT_PASSWORD = '$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYcl'\
|
||||
'wHqMq9e3Tg8rvQxhxSlBXP1DZhdUamxdOBXK0.'
|
||||
|
||||
|
||||
def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False,
|
||||
attempts=CONF.fetch_packages_attempts):
|
||||
"""Builds initial base system.
|
||||
|
||||
debootstrap builds initial base system which is capable to run apt-get.
|
||||
debootstrap is well known for its glithcy resolving of package dependecies,
|
||||
so the rest of packages will be installed later by run_apt_get.
|
||||
"""
|
||||
cmds = ['debootstrap', '--verbose', '--no-check-gpg', '--arch=%s' % arch,
|
||||
suite, chroot, uri]
|
||||
if eatmydata:
|
||||
cmds.insert(4, '--include=eatmydata')
|
||||
stdout, stderr = utils.execute(*cmds, attempts=attempts)
|
||||
LOG.debug('Running deboostrap completed.\nstdout: %s\nstderr: %s', stdout,
|
||||
stderr)
|
||||
|
||||
|
||||
def set_apt_get_env():
|
||||
# NOTE(agordeev): disable any confirmations/questions from apt-get side
|
||||
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
|
||||
os.environ['DEBCONF_NONINTERACTIVE_SEEN'] = 'true'
|
||||
os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C'
|
||||
|
||||
|
||||
def run_apt_get(chroot, packages, eatmydata=False,
|
||||
attempts=CONF.fetch_packages_attempts):
|
||||
"""Runs apt-get install <packages>.
|
||||
|
||||
Unlike debootstrap, apt-get has a perfect package dependecies resolver
|
||||
under the hood.
|
||||
eatmydata could be used to totally ignore the storm of sync() calls from
|
||||
dpkg/apt-get tools. It's dangerous, but could decrease package install
|
||||
time in X times.
|
||||
"""
|
||||
cmds = ['chroot', chroot, 'apt-get', '-y', 'update']
|
||||
stdout, stderr = utils.execute(*cmds, attempts=attempts)
|
||||
LOG.debug('Running apt-get update completed.\nstdout: %s\nstderr: %s',
|
||||
stdout, stderr)
|
||||
cmds = ['chroot', chroot, 'apt-get', '-y', 'install', ' '.join(packages)]
|
||||
if eatmydata:
|
||||
cmds.insert(2, 'eatmydata')
|
||||
stdout, stderr = utils.execute(*cmds, attempts=attempts)
|
||||
LOG.debug('Running apt-get install completed.\nstdout: %s\nstderr: %s',
|
||||
stdout, stderr)
|
||||
|
||||
|
||||
def suppress_services_start(chroot):
|
||||
"""Suppresses services start.
|
||||
|
||||
Prevents start of any service such as udev/ssh/etc in chrooted environment
|
||||
while image is being built.
|
||||
"""
|
||||
path = os.path.join(chroot, 'usr/sbin')
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
with open(os.path.join(path, 'policy-rc.d'), 'w') as f:
|
||||
f.write('#!/bin/sh\n'
|
||||
'# prevent any service from being started\n'
|
||||
'exit 101\n')
|
||||
os.fchmod(f.fileno(), 0o755)
|
||||
|
||||
|
||||
def clean_dirs(chroot, dirs, delete=False):
|
||||
"""Removes dirs and recreates them
|
||||
|
||||
:param chroot: Root directory where to look for subdirectories
|
||||
:param dirs: List of directories to clean/remove (Relative to chroot)
|
||||
:param delete: (Boolean) If True, directories will be removed
|
||||
(Default: False)
|
||||
"""
|
||||
for d in dirs:
|
||||
path = os.path.join(chroot, d)
|
||||
if os.path.isdir(path):
|
||||
LOG.debug('Removing dir: %s', path)
|
||||
shutil.rmtree(path)
|
||||
if not delete:
|
||||
LOG.debug('Creating empty dir: %s', path)
|
||||
os.makedirs(path)
|
||||
|
||||
|
||||
def remove_files(chroot, files):
|
||||
for f in files:
|
||||
path = os.path.join(chroot, f)
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
LOG.debug('Removed file: %s', path)
|
||||
|
||||
|
||||
def clean_apt_settings(chroot, allow_unsigned_file=CONF.allow_unsigned_file,
|
||||
force_ipv4_file=CONF.force_ipv4_file):
|
||||
"""Cleans apt settings such as package sources and repo pinning."""
|
||||
files = [DEFAULT_APT_PATH['sources_file'],
|
||||
DEFAULT_APT_PATH['preferences_file'],
|
||||
os.path.join(DEFAULT_APT_PATH['conf_dir'], force_ipv4_file),
|
||||
os.path.join(DEFAULT_APT_PATH['conf_dir'], allow_unsigned_file)]
|
||||
remove_files(chroot, files)
|
||||
dirs = [DEFAULT_APT_PATH['preferences_dir'],
|
||||
DEFAULT_APT_PATH['sources_dir']]
|
||||
clean_dirs(chroot, dirs)
|
||||
|
||||
|
||||
def do_post_inst(chroot):
|
||||
# NOTE(agordeev): set up password for root
|
||||
utils.execute('sed', '-i',
|
||||
's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
|
||||
os.path.join(chroot, 'etc/shadow'))
|
||||
# NOTE(agordeev): backport from bash-script:
|
||||
# in order to prevent the later puppet workflow outage, puppet service
|
||||
# should be disabled on a node startup.
|
||||
# Being enabled by default, sometimes it leads to puppet service hanging
|
||||
# and recognizing the deployment as failed.
|
||||
# TODO(agordeev): take care of puppet service for other distros, once
|
||||
# fuel-agent will be capable of building images for them too.
|
||||
utils.execute('chroot', chroot, 'update-rc.d', 'puppet', 'disable')
|
||||
# NOTE(agordeev): disable mcollective to be automatically started on boot
|
||||
# to prevent confusing messages in its log (regarding connection errors).
|
||||
with open(os.path.join(chroot, 'etc/init/mcollective.override'), 'w') as f:
|
||||
f.write("manual\n")
|
||||
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
|
||||
# execution of post/pre-install package hooks and start of services
|
||||
remove_files(chroot, ['usr/sbin/policy-rc.d'])
|
||||
clean_apt_settings(chroot)
|
||||
|
||||
|
||||
def stop_chrooted_processes(chroot, signal=sig.SIGTERM,
|
||||
attempts=10, attempts_delay=2):
|
||||
"""Sends signal to all processes, which are running inside chroot.
|
||||
|
||||
It tries several times until all processes die. If at some point there
|
||||
are no running processes found, it returns True.
|
||||
|
||||
:param chroot: Process root directory.
|
||||
:param signal: Which signal to send to processes. It must be either
|
||||
SIGTERM or SIGKILL. (Default: SIGTERM)
|
||||
:param attempts: Number of attempts (Default: 10)
|
||||
:param attempts_delay: Delay between attempts (Default: 2)
|
||||
"""
|
||||
|
||||
if signal not in (sig.SIGTERM, sig.SIGKILL):
|
||||
raise ValueError('Signal must be either SIGTERM or SIGKILL')
|
||||
|
||||
def get_running_processes():
|
||||
return utils.execute(
|
||||
'fuser', '-v', chroot, check_exit_code=False)[0].split()
|
||||
|
||||
for i in six.moves.range(attempts):
|
||||
running_processes = get_running_processes()
|
||||
if not running_processes:
|
||||
LOG.debug('There are no running processes in %s ', chroot)
|
||||
return True
|
||||
for p in running_processes:
|
||||
try:
|
||||
pid = int(p)
|
||||
if os.readlink('/proc/%s/root' % pid) == chroot:
|
||||
LOG.debug('Sending %s to chrooted process %s', signal, pid)
|
||||
os.kill(pid, signal)
|
||||
except (OSError, ValueError) as e:
|
||||
cmdline = ''
|
||||
pid = p
|
||||
try:
|
||||
with open('/proc/%s/cmdline' % pid) as f:
|
||||
cmdline = f.read()
|
||||
except Exception:
|
||||
LOG.debug('Can not read cmdline for pid=%s', pid)
|
||||
LOG.warning('Exception while sending signal: '
|
||||
'pid: %s cmdline: %s message: %s. Skipping it.',
|
||||
pid, cmdline, e)
|
||||
|
||||
# First of all, signal delivery is asynchronous.
|
||||
# Just because the signal has been sent doesn't
|
||||
# mean the kernel will deliver it instantly
|
||||
# (the target process might be uninterruptible at the moment).
|
||||
# Secondly, exiting might take a while (the process might have
|
||||
# some data to fsync, etc)
|
||||
LOG.debug('Attempt %s. Waiting for %s seconds', i + 1, attempts_delay)
|
||||
time.sleep(attempts_delay)
|
||||
|
||||
running_processes = get_running_processes()
|
||||
if running_processes:
|
||||
for pid in running_processes:
|
||||
cmdline = ''
|
||||
try:
|
||||
with open('/proc/%s/cmdline' % pid) as f:
|
||||
cmdline = f.read()
|
||||
except Exception:
|
||||
LOG.debug('Can not read cmdline for pid=%s', pid)
|
||||
LOG.warning('Process is still running: pid=%s cmdline: %s',
|
||||
pid, cmdline)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_free_loop_device(
|
||||
loop_device_major_number=CONF.loop_device_major_number,
|
||||
max_loop_devices_count=CONF.max_loop_devices_count):
|
||||
"""Returns the name of free loop device.
|
||||
|
||||
It should return the name of free loop device or raise an exception.
|
||||
Unfortunately, free loop device couldn't be reversed for the later usage,
|
||||
so we must start to use it as fast as we can.
|
||||
If there's no free loop it will try to create new one and ask a system for
|
||||
free loop again.
|
||||
"""
|
||||
for minor in range(0, max_loop_devices_count):
|
||||
cur_loop = "/dev/loop%s" % minor
|
||||
if not os.path.exists(cur_loop):
|
||||
os.mknod(cur_loop, 0o660 | stat.S_IFBLK,
|
||||
os.makedev(loop_device_major_number, minor))
|
||||
try:
|
||||
return utils.execute('losetup', '--find')[0].split()[0]
|
||||
except (IndexError, errors.ProcessExecutionError):
|
||||
LOG.debug("Couldn't find free loop device, trying again")
|
||||
raise errors.NoFreeLoopDevices('Free loop device not found')
|
||||
|
||||
|
||||
def create_sparse_tmp_file(dir, suffix, size=CONF.sparse_file_size):
|
||||
"""Creates sparse file.
|
||||
|
||||
Creates file which consumes disk space more efficiently when the file
|
||||
itself is mostly empty.
|
||||
"""
|
||||
tf = tempfile.NamedTemporaryFile(dir=dir, suffix=suffix, delete=False)
|
||||
utils.execute('truncate', '-s', '%sM' % size, tf.name)
|
||||
return tf.name
|
||||
|
||||
|
||||
def attach_file_to_loop(filename, loop):
|
||||
utils.execute('losetup', loop, filename)
|
||||
|
||||
|
||||
def deattach_loop(loop, check_exit_code=[0]):
|
||||
LOG.debug('Trying to figure out if loop device %s is attached', loop)
|
||||
output = utils.execute('losetup', '-a')[0]
|
||||
for line in output.split('\n'):
|
||||
# output lines are assumed to have the following format
|
||||
# /dev/loop0: [fd03]:130820 (/dev/loop0)
|
||||
if loop == line.split(':')[0]:
|
||||
LOG.debug('Loop device %s seems to be attached. '
|
||||
'Trying to detach.', loop)
|
||||
utils.execute('losetup', '-d', loop,
|
||||
check_exit_code=check_exit_code)
|
||||
|
||||
|
||||
def shrink_sparse_file(filename):
|
||||
"""Shrinks file to its size of actual data. Only ext fs are supported."""
|
||||
utils.execute('e2fsck', '-y', '-f', filename)
|
||||
utils.execute('resize2fs', '-F', '-M', filename)
|
||||
data = hu.parse_simple_kv('dumpe2fs', filename)
|
||||
block_count = int(data['block count'])
|
||||
block_size = int(data['block size'])
|
||||
with open(filename, 'rwb+') as f:
|
||||
f.truncate(block_count * block_size)
|
||||
|
||||
|
||||
def strip_filename(name):
|
||||
"""Strips filename for apt settings.
|
||||
|
||||
The name could only contain alphanumeric, hyphen (-), underscore (_) and
|
||||
period (.) characters.
|
||||
"""
|
||||
return re.sub(r"[^a-zA-Z0-9-_.]*", "", name)
|
||||
|
||||
|
||||
def get_release_file(uri, suite, section):
|
||||
"""Download and parse repo's Release file
|
||||
|
||||
It and returns an apt preferences line for specified repo.
|
||||
|
||||
:param repo: a repo as dict
|
||||
:returns: a string with apt preferences rules
|
||||
"""
|
||||
if section:
|
||||
# We can't use urljoin here because it works pretty bad in
|
||||
# cases when 'uri' doesn't have a trailing slash.
|
||||
download_uri = os.path.join(uri, 'dists', suite, 'Release')
|
||||
else:
|
||||
# Well, we have a flat repo case, so we should download Release
|
||||
# file from a different place. Please note, we have to strip
|
||||
# a leading slash from suite because otherwise the download
|
||||
# link will be wrong.
|
||||
download_uri = os.path.join(uri, suite.lstrip('/'), 'Release')
|
||||
|
||||
return utils.init_http_request(download_uri).text
|
||||
|
||||
|
||||
def parse_release_file(content):
|
||||
"""Parse Debian repo's Release file content.
|
||||
|
||||
:param content: a Debian's Release file content
|
||||
:returns: a dict with repo's attributes
|
||||
"""
|
||||
_multivalued_fields = {
|
||||
'SHA1': ['sha1', 'size', 'name'],
|
||||
'SHA256': ['sha256', 'size', 'name'],
|
||||
'SHA512': ['sha512', 'size', 'name'],
|
||||
'MD5Sum': ['md5sum', 'size', 'name'],
|
||||
}
|
||||
|
||||
# debian data format is very similiar to yaml, except
|
||||
# multivalued field. so we can parse it just like yaml
|
||||
# and then perform additional transformation for those
|
||||
# fields (we know which ones are multivalues).
|
||||
data = yaml.load(content)
|
||||
|
||||
for attr, columns in six.iteritems(_multivalued_fields):
|
||||
if attr not in data:
|
||||
continue
|
||||
|
||||
values = data[attr].split()
|
||||
data[attr] = []
|
||||
|
||||
for group in utils.grouper(values, len(columns)):
|
||||
data[attr].append(dict(zip(columns, group)))
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def add_apt_source(name, uri, suite, section, chroot):
|
||||
# NOTE(agordeev): The files have either no or "list" as filename extension
|
||||
filename = 'fuel-image-{name}.list'.format(name=strip_filename(name))
|
||||
if section:
|
||||
entry = 'deb {uri} {suite} {section}\n'.format(uri=uri, suite=suite,
|
||||
section=section)
|
||||
else:
|
||||
entry = 'deb {uri} {suite}\n'.format(uri=uri, suite=suite)
|
||||
with open(os.path.join(chroot, DEFAULT_APT_PATH['sources_dir'], filename),
|
||||
'w') as f:
|
||||
f.write(entry)
|
||||
|
||||
|
||||
def add_apt_preference(name, priority, suite, section, chroot, uri):
|
||||
# NOTE(agordeev): The files have either no or "pref" as filename extension
|
||||
filename = 'fuel-image-{name}.pref'.format(name=strip_filename(name))
|
||||
# NOTE(agordeev): priotity=None means that there's no specific pinning for
|
||||
# particular repo and nothing to process.
|
||||
# Default system-wide preferences (priority=500) will be used instead.
|
||||
|
||||
_transformations = {
|
||||
'Archive': 'a',
|
||||
'Suite': 'a', # suite is a synonym for archive
|
||||
'Codename': 'n',
|
||||
'Version': 'v',
|
||||
'Origin': 'o',
|
||||
'Label': 'l',
|
||||
}
|
||||
|
||||
try:
|
||||
deb_release = parse_release_file(
|
||||
get_release_file(uri, suite, section)
|
||||
)
|
||||
except ValueError as exc:
|
||||
LOG.error(
|
||||
"[Attention] Failed to fetch Release file "
|
||||
"for repo '{0}': {1} - skipping. "
|
||||
"This may lead both to trouble with packages "
|
||||
"and broken OS".format(name, six.text_type(exc))
|
||||
)
|
||||
return
|
||||
|
||||
conditions = set()
|
||||
for field, condition in six.iteritems(_transformations):
|
||||
if field in deb_release:
|
||||
conditions.add(
|
||||
'{0}={1}'.format(condition, deb_release[field])
|
||||
)
|
||||
|
||||
with open(os.path.join(chroot, DEFAULT_APT_PATH['preferences_dir'],
|
||||
filename), 'w') as f:
|
||||
sections = section.split()
|
||||
if sections:
|
||||
for s in sections:
|
||||
f.write('Package: *\n')
|
||||
f.write('Pin: release ')
|
||||
f.write(', '.join(conditions) + ", c={0}\n".format(s))
|
||||
f.write('Pin-Priority: {priority}\n'.format(priority=priority))
|
||||
else:
|
||||
f.write('Package: *\n')
|
||||
f.write('Pin: release ')
|
||||
f.write(', '.join(conditions) + "\n")
|
||||
f.write('Pin-Priority: {priority}\n'.format(priority=priority))
|
||||
|
||||
|
||||
def pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file,
|
||||
force_ipv4_file=CONF.force_ipv4_file):
|
||||
"""It must be called prior run_apt_get."""
|
||||
clean_apt_settings(chroot)
|
||||
# NOTE(agordeev): allow to install packages without gpg digest
|
||||
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
|
||||
allow_unsigned_file), 'w') as f:
|
||||
f.write('APT::Get::AllowUnauthenticated 1;\n')
|
||||
with open(os.path.join(chroot, DEFAULT_APT_PATH['conf_dir'],
|
||||
force_ipv4_file), 'w') as f:
|
||||
f.write('Acquire::ForceIPv4 "true";\n')
|
||||
|
||||
|
||||
def containerize(filename, container, chunk_size=CONF.data_chunk_size):
|
||||
if container == 'gzip':
|
||||
output_file = filename + '.gz'
|
||||
with open(filename, 'rb') as f:
|
||||
# NOTE(agordeev): gzip in python2.6 doesn't have context manager
|
||||
# support
|
||||
g = gzip.open(output_file, 'wb')
|
||||
for chunk in iter(lambda: f.read(chunk_size), ''):
|
||||
g.write(chunk)
|
||||
g.close()
|
||||
os.remove(filename)
|
||||
return output_file
|
||||
raise errors.WrongImageDataError(
|
||||
'Error while image initialization: '
|
||||
'unsupported image container: {container}'.format(container=container))
|
|
@ -1,78 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_fs(fs_type, fs_options, fs_label, dev):
|
||||
# NOTE(agordeev): notice the different flag to force the fs creating
|
||||
# ext* uses -F flag, xfs/mkswap uses -f flag.
|
||||
cmd_line = []
|
||||
cmd_name = 'mkswap'
|
||||
if fs_type != 'swap':
|
||||
cmd_name = 'mkfs.%s' % fs_type
|
||||
cmd_line.append(cmd_name)
|
||||
for opt in (fs_options, fs_label):
|
||||
cmd_line.extend([s for s in opt.split(' ') if s])
|
||||
cmd_line.append(dev)
|
||||
utils.execute(*cmd_line)
|
||||
|
||||
|
||||
def extend_fs(fs_type, fs_dev):
|
||||
if fs_type in ('ext2', 'ext3', 'ext4'):
|
||||
# ext3,4 file system can be mounted
|
||||
# must be checked with e2fsck -f
|
||||
utils.execute('e2fsck', '-yf', fs_dev, check_exit_code=[0])
|
||||
utils.execute('resize2fs', fs_dev, check_exit_code=[0])
|
||||
utils.execute('e2fsck', '-pf', fs_dev, check_exit_code=[0])
|
||||
elif fs_type == 'xfs':
|
||||
# xfs file system must be mounted
|
||||
utils.execute('xfs_growfs', fs_dev, check_exit_code=[0])
|
||||
else:
|
||||
raise errors.FsUtilsError('Unsupported file system type')
|
||||
|
||||
|
||||
def mount_fs(fs_type, fs_dev, fs_mount):
|
||||
utils.execute('mount', '-t', fs_type, fs_dev, fs_mount,
|
||||
check_exit_code=[0])
|
||||
|
||||
|
||||
def mount_bind(chroot, path, path2=None):
|
||||
if not path2:
|
||||
path2 = path
|
||||
utils.execute('mount', '--bind', path, chroot + path2,
|
||||
check_exit_code=[0])
|
||||
|
||||
|
||||
def umount_fs(fs_mount, try_lazy_umount=False):
|
||||
try:
|
||||
utils.execute('mountpoint', '-q', fs_mount, check_exit_code=[0])
|
||||
except errors.ProcessExecutionError:
|
||||
LOG.warning('%s is not a mountpoint, skipping umount', fs_mount)
|
||||
else:
|
||||
LOG.debug('Trying to umount {0}'.format(fs_mount))
|
||||
try:
|
||||
utils.execute('umount', fs_mount, check_exit_code=[0])
|
||||
except errors.ProcessExecutionError as e:
|
||||
if try_lazy_umount:
|
||||
LOG.warning('Error while umounting {0} '
|
||||
'exc={1}'.format(fs_mount, e.message))
|
||||
LOG.debug('Trying lazy umounting {0}'.format(fs_mount))
|
||||
utils.execute('umount', '-l', fs_mount, check_exit_code=[0])
|
||||
else:
|
||||
raise
|
|
@ -1,257 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
gu_opts = [
|
||||
cfg.IntOpt(
|
||||
'grub_timeout',
|
||||
default=5,
|
||||
help='Timeout in secs for GRUB'
|
||||
),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(gu_opts)
|
||||
|
||||
|
||||
def guess_grub2_conf(chroot=''):
|
||||
for filename in ('/boot/grub/grub.cfg', '/boot/grub2/grub.cfg'):
|
||||
if os.path.isdir(os.path.dirname(chroot + filename)):
|
||||
return filename
|
||||
|
||||
|
||||
def guess_grub2_default(chroot=''):
|
||||
for filename in ('/etc/default/grub', '/etc/sysconfig/grub'):
|
||||
if os.path.isfile(chroot + filename):
|
||||
return filename
|
||||
|
||||
|
||||
def guess_grub2_mkconfig(chroot=''):
|
||||
for grub_mkconfig in \
|
||||
('/sbin/grub-mkconfig', '/sbin/grub2-mkconfig',
|
||||
'/usr/sbin/grub-mkconfig', '/usr/sbin/grub2-mkconfig'):
|
||||
if os.path.isfile(chroot + grub_mkconfig):
|
||||
return grub_mkconfig
|
||||
|
||||
|
||||
def guess_grub_version(chroot=''):
|
||||
grub_install = guess_grub_install(chroot=chroot)
|
||||
LOG.debug('Trying to run %s --version' % grub_install)
|
||||
cmd = [grub_install, '--version']
|
||||
if chroot:
|
||||
cmd[:0] = ['chroot', chroot]
|
||||
result = utils.execute(*cmd)
|
||||
version = 1 if result[0].find('0.97') > 0 else 2
|
||||
LOG.debug('Looks like grub version is %s' % version)
|
||||
return version
|
||||
|
||||
|
||||
def guess_grub(chroot=''):
|
||||
for grub in ('/sbin/grub', '/usr/sbin/grub'):
|
||||
LOG.debug('Looking for grub: trying %s' % grub)
|
||||
if os.path.isfile(chroot + grub):
|
||||
LOG.debug('grub found: %s' % grub)
|
||||
return grub
|
||||
raise errors.GrubUtilsError('grub not found')
|
||||
|
||||
|
||||
def guess_grub_install(chroot=''):
|
||||
for grub_install in ('/sbin/grub-install', '/sbin/grub2-install',
|
||||
'/usr/sbin/grub-install', '/usr/sbin/grub2-install'):
|
||||
LOG.debug('Looking for grub-install: trying %s' % grub_install)
|
||||
if os.path.isfile(chroot + grub_install):
|
||||
LOG.debug('grub-install found: %s' % grub_install)
|
||||
return grub_install
|
||||
raise errors.GrubUtilsError('grub-install not found')
|
||||
|
||||
|
||||
def guess_grub1_datadir(chroot='', arch='x86_64'):
|
||||
LOG.debug('Looking for grub data directory')
|
||||
for d in os.listdir(chroot + '/usr/share/grub'):
|
||||
if arch in d:
|
||||
LOG.debug('Looks like grub data directory '
|
||||
'is /usr/share/grub/%s' % d)
|
||||
return '/usr/share/grub/' + d
|
||||
|
||||
|
||||
def guess_kernel(chroot='', regexp=None):
|
||||
"""Tries to guess kernel by regexp
|
||||
|
||||
:param chroot: Path to chroot
|
||||
:param regexp: (String) Regular expression (must have python syntax).
|
||||
Default is r'^vmlinuz.*'
|
||||
"""
|
||||
kernel = utils.guess_filename(
|
||||
path=os.path.join(chroot, 'boot'),
|
||||
regexp=(regexp or r'^vmlinuz.*'))
|
||||
|
||||
if kernel:
|
||||
return kernel
|
||||
|
||||
raise errors.GrubUtilsError('Error while trying to find kernel: '
|
||||
'regexp=%s' % regexp)
|
||||
|
||||
|
||||
def guess_initrd(chroot='', regexp=None):
|
||||
"""Tries to guess initrd by regexp
|
||||
|
||||
:param chroot: Path to chroot
|
||||
:param regexp: (String) Regular expression (must have python syntax).
|
||||
Default is r'^(initrd|initramfs).*'
|
||||
"""
|
||||
initrd = utils.guess_filename(
|
||||
path=os.path.join(chroot, 'boot'),
|
||||
regexp=(regexp or r'^(initrd|initramfs).*'))
|
||||
|
||||
if initrd:
|
||||
return initrd
|
||||
|
||||
raise errors.GrubUtilsError('Error while trying to find initrd: '
|
||||
'regexp=%s' % regexp)
|
||||
|
||||
|
||||
def grub1_install(install_devices, boot_device, chroot=''):
|
||||
match = re.search(r'(.+?)(p?)(\d*)$', boot_device)
|
||||
# Checking whether boot device is a partition
|
||||
# !!! It must be a partition not a whole disk. !!!
|
||||
if not match.group(3):
|
||||
raise errors.GrubUtilsError(
|
||||
'Error while installing legacy grub: '
|
||||
'boot device must be a partition')
|
||||
boot_disk = match.group(1)
|
||||
boot_part = str(int(match.group(3)) - 1)
|
||||
grub1_stage1(chroot=chroot)
|
||||
for install_device in install_devices:
|
||||
grub1_mbr(install_device, boot_disk, boot_part, chroot=chroot)
|
||||
|
||||
|
||||
def grub1_mbr(install_device, boot_disk, boot_part, chroot=''):
|
||||
# The device on which we are going to install
|
||||
# stage1 needs to be mapped as hd0, otherwise system won't be able to boot.
|
||||
batch = 'device (hd0) {0}\n'.format(install_device)
|
||||
# That is much easier to use grub-install, but unfortunately
|
||||
# it is not able to install bootloader on huge disks.
|
||||
# Instead we set drive geometry manually to avoid grub register
|
||||
# overlapping. We set it so as to make grub
|
||||
# thinking that disk size is equal to 1G.
|
||||
# 130 cylinders * (16065 * 512 = 8225280 bytes) = 1G
|
||||
# We also assume that boot partition is in the beginning
|
||||
# of disk between 0 and 1G.
|
||||
batch += 'geometry (hd0) 130 255 63\n'
|
||||
if boot_disk != install_device:
|
||||
batch += 'device (hd1) {0}\n'.format(boot_disk)
|
||||
batch += 'geometry (hd1) 130 255 63\n'
|
||||
batch += 'root (hd1,{0})\n'.format(boot_part)
|
||||
else:
|
||||
batch += 'root (hd0,{0})\n'.format(boot_part)
|
||||
batch += 'setup (hd0)\n'
|
||||
batch += 'quit\n'
|
||||
|
||||
with open(chroot + '/tmp/grub.batch', 'wb') as f:
|
||||
LOG.debug('Grub batch content: \n%s' % batch)
|
||||
f.write(batch)
|
||||
|
||||
script = 'cat /tmp/grub.batch | {0} --no-floppy --batch'.format(
|
||||
guess_grub(chroot=chroot))
|
||||
with open(chroot + '/tmp/grub.sh', 'wb') as f:
|
||||
LOG.debug('Grub script content: \n%s' % script)
|
||||
f.write(script)
|
||||
|
||||
os.chmod(chroot + '/tmp/grub.sh', 0o755)
|
||||
cmd = ['/tmp/grub.sh']
|
||||
if chroot:
|
||||
cmd[:0] = ['chroot', chroot]
|
||||
stdout, stderr = utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
|
||||
LOG.debug('Grub script stdout: \n%s' % stdout)
|
||||
LOG.debug('Grub script stderr: \n%s' % stderr)
|
||||
|
||||
|
||||
def grub1_stage1(chroot=''):
|
||||
LOG.debug('Installing grub stage1 files')
|
||||
for f in os.listdir(chroot + '/boot/grub'):
|
||||
if f in ('stage1', 'stage2') or 'stage1_5' in f:
|
||||
LOG.debug('Removing: %s' % chroot + os.path.join('/boot/grub', f))
|
||||
os.remove(chroot + os.path.join('/boot/grub', f))
|
||||
grub1_datadir = guess_grub1_datadir(chroot=chroot)
|
||||
for f in os.listdir(chroot + grub1_datadir):
|
||||
if f in ('stage1', 'stage2') or 'stage1_5' in f:
|
||||
LOG.debug('Copying %s from %s to /boot/grub' % (f, grub1_datadir))
|
||||
shutil.copy(chroot + os.path.join(grub1_datadir, f),
|
||||
chroot + os.path.join('/boot/grub', f))
|
||||
|
||||
|
||||
def grub1_cfg(kernel=None, initrd=None,
|
||||
kernel_params='', chroot='', grub_timeout=CONF.grub_timeout):
|
||||
|
||||
if not kernel:
|
||||
kernel = guess_kernel(chroot=chroot)
|
||||
if not initrd:
|
||||
initrd = guess_initrd(chroot=chroot)
|
||||
|
||||
config = """
|
||||
default=0
|
||||
timeout={grub_timeout}
|
||||
title Default ({kernel})
|
||||
kernel /{kernel} {kernel_params}
|
||||
initrd /{initrd}
|
||||
""".format(kernel=kernel, initrd=initrd,
|
||||
kernel_params=kernel_params,
|
||||
grub_timeout=grub_timeout)
|
||||
with open(chroot + '/boot/grub/grub.conf', 'wb') as f:
|
||||
f.write(config)
|
||||
|
||||
|
||||
def grub2_install(install_devices, chroot=''):
|
||||
grub_install = guess_grub_install(chroot=chroot)
|
||||
for install_device in install_devices:
|
||||
cmd = [grub_install, install_device]
|
||||
if chroot:
|
||||
cmd[:0] = ['chroot', chroot]
|
||||
utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
|
||||
|
||||
|
||||
def grub2_cfg(kernel_params='', chroot='', grub_timeout=CONF.grub_timeout):
|
||||
grub_defaults = chroot + guess_grub2_default(chroot=chroot)
|
||||
rekerparams = re.compile(r'^.*GRUB_CMDLINE_LINUX=.*')
|
||||
retimeout = re.compile(r'^.*GRUB_HIDDEN_TIMEOUT=.*')
|
||||
new_content = ''
|
||||
with open(grub_defaults) as f:
|
||||
for line in f:
|
||||
line = rekerparams.sub(
|
||||
'GRUB_CMDLINE_LINUX="{kernel_params}"'.
|
||||
format(kernel_params=kernel_params), line)
|
||||
line = retimeout.sub('GRUB_HIDDEN_TIMEOUT={grub_timeout}'.
|
||||
format(grub_timeout=grub_timeout), line)
|
||||
new_content += line
|
||||
# NOTE(agordeev): explicitly add record fail timeout, in order to
|
||||
# prevent user confirmation appearing if unexpected reboot occured.
|
||||
new_content += '\nGRUB_RECORDFAIL_TIMEOUT={grub_timeout}\n'.\
|
||||
format(grub_timeout=grub_timeout)
|
||||
with open(grub_defaults, 'wb') as f:
|
||||
f.write(new_content)
|
||||
cmd = [guess_grub2_mkconfig(chroot), '-o', guess_grub2_conf(chroot)]
|
||||
if chroot:
|
||||
cmd[:0] = ['chroot', chroot]
|
||||
utils.execute(*cmd, run_as_root=True)
|
|
@ -1,373 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
# Please take a look at the linux kernel documentation
|
||||
# https://github.com/torvalds/linux/blob/master/Documentation/devices.txt.
|
||||
# KVM virtio volumes have major number 252 in CentOS, but 253 in Ubuntu.
|
||||
# NOTE(agordeev): nvme devices also have a major number of 259
|
||||
# (only in 2.6 kernels)
|
||||
VALID_MAJORS = (3, 8, 65, 66, 67, 68, 69, 70, 71, 104, 105, 106, 107, 108, 109,
|
||||
110, 111, 202, 252, 253, 259)
|
||||
|
||||
# We are only interested in getting these
|
||||
# properties from udevadm report
|
||||
# MAJOR major device number
|
||||
# MINOR minor device number
|
||||
# DEVNAME e.g. /dev/sda
|
||||
# DEVTYPE e.g. disk or partition for block devices
|
||||
# DEVPATH path to a device directory relative to /sys
|
||||
# ID_BUS e.g. ata, scsi
|
||||
# ID_MODEL e.g. MATSHITADVD-RAM_UJ890
|
||||
# ID_SERIAL_SHORT e.g. UH00_296679
|
||||
# ID_WWN e.g. 0x50000392e9804d4b (optional)
|
||||
# ID_CDROM e.g. 1 for cdrom device (optional)
|
||||
UDEV_PROPERTIES = set(['MAJOR', 'MINOR', 'DEVNAME', 'DEVTYPE', 'DEVPATH',
|
||||
'ID_BUS', 'ID_MODEL', 'ID_SERIAL_SHORT', 'ID_WWN',
|
||||
'ID_CDROM', 'ID_VENDOR'])
|
||||
REMOVABLE_VENDORS = [
|
||||
"Adaptec", "IBM", "ServeRA",
|
||||
]
|
||||
|
||||
# more details about types you can find in dmidecode's manual
|
||||
SMBIOS_TYPES = {'bios': '0',
|
||||
'base_board': '2',
|
||||
'processor': '4',
|
||||
'memory_array': '16',
|
||||
'memory_device': '17'}
|
||||
|
||||
|
||||
def parse_dmidecode(type):
|
||||
"""Parses `dmidecode` output.
|
||||
|
||||
:param type: A string with type of entity to display.
|
||||
|
||||
:returns: A list with dictionaries of entities for specified type.
|
||||
"""
|
||||
output = utils.execute('dmidecode', '-q', '--type', type)
|
||||
lines = output[0].split('\n')
|
||||
info = []
|
||||
multiline_values = None
|
||||
section = 0
|
||||
|
||||
for line in lines:
|
||||
if len(line) != 0 and len(line.strip()) == len(line):
|
||||
info.append({})
|
||||
section = len(info) - 1
|
||||
try:
|
||||
k, v = (l.strip() for l in line.split(':', 1))
|
||||
except ValueError:
|
||||
k = line.strip()
|
||||
if not k:
|
||||
multiline_values = None
|
||||
if multiline_values:
|
||||
info[section][multiline_values].append(k)
|
||||
else:
|
||||
if not v:
|
||||
multiline_values = k.lower()
|
||||
info[section][multiline_values] = []
|
||||
else:
|
||||
info[section][k.lower()] = v
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def parse_lspci():
|
||||
"""Parses `lspci` output.
|
||||
|
||||
:returns: A list of dicts containing PCI devices information
|
||||
"""
|
||||
output = utils.execute('lspci', '-vmm', '-D')
|
||||
lines = output[0].split('\n')
|
||||
info = [{}]
|
||||
section = 0
|
||||
|
||||
for line in lines[:-2]:
|
||||
try:
|
||||
k, v = (l.strip() for l in line.split(':', 1))
|
||||
except ValueError:
|
||||
info.append({})
|
||||
section += 1
|
||||
else:
|
||||
info[section][k.lower()] = v
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def parse_simple_kv(*command):
|
||||
"""Parses simple key:value output from specified command.
|
||||
|
||||
:param command: A command to execute
|
||||
|
||||
:returns: A dict of parsed key-value data
|
||||
"""
|
||||
output = utils.execute(*command)
|
||||
lines = output[0].split('\n')
|
||||
info = {}
|
||||
|
||||
for line in lines[:-1]:
|
||||
try:
|
||||
k, v = (l.strip() for l in line.split(':', 1))
|
||||
except ValueError:
|
||||
break
|
||||
else:
|
||||
info[k.lower()] = v
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def is_disk(dev, bspec=None, uspec=None):
|
||||
"""Checks if given device is a disk.
|
||||
|
||||
:param dev: A device file, e.g. /dev/sda.
|
||||
:param bspec: A dict of properties which we get from blockdev.
|
||||
:param uspec: A dict of properties which we get from udevadm.
|
||||
|
||||
:returns: True if device is disk else False.
|
||||
"""
|
||||
|
||||
# Filtering by udevspec
|
||||
if uspec is None:
|
||||
uspec = udevreport(dev)
|
||||
if uspec.get('ID_CDROM') == '1':
|
||||
return False
|
||||
if uspec.get('DEVTYPE') == 'partition':
|
||||
return False
|
||||
if 'MAJOR' in uspec and int(uspec['MAJOR']) not in VALID_MAJORS:
|
||||
return False
|
||||
|
||||
# Filtering by blockdev spec
|
||||
if bspec is None:
|
||||
bspec = blockdevreport(dev)
|
||||
if bspec.get('ro') == '1':
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def udevreport(dev):
|
||||
"""Builds device udevadm report.
|
||||
|
||||
:param dev: A device file, e.g. /dev/sda.
|
||||
|
||||
:returns: A dict of udev device properties.
|
||||
"""
|
||||
report = utils.execute('udevadm',
|
||||
'info',
|
||||
'--query=property',
|
||||
'--export',
|
||||
'--name={0}'.format(dev),
|
||||
check_exit_code=[0])[0]
|
||||
|
||||
spec = {}
|
||||
for line in [l for l in report.splitlines() if l]:
|
||||
key, value = line.split('=', 1)
|
||||
value = value.strip('\'')
|
||||
|
||||
# This is a list of symbolic links which were created for this
|
||||
# block device (e.g. /dev/disk/by-id/foobar)
|
||||
if key == 'DEVLINKS':
|
||||
spec['DEVLINKS'] = value.split()
|
||||
|
||||
if key in UDEV_PROPERTIES:
|
||||
spec[key] = value
|
||||
return spec
|
||||
|
||||
|
||||
def blockdevreport(blockdev):
|
||||
"""Builds device blockdev report.
|
||||
|
||||
:param blockdev: A block device file, e.g. /dev/sda.
|
||||
|
||||
:returns: A dict of blockdev properties.
|
||||
"""
|
||||
cmd = [
|
||||
'blockdev',
|
||||
'--getsz', # get size in 512-byte sectors
|
||||
'--getro', # get read-only
|
||||
'--getss', # get logical block (sector) size
|
||||
'--getpbsz', # get physical block (sector) size
|
||||
'--getsize64', # get size in bytes
|
||||
'--getiomin', # get minimum I/O size
|
||||
'--getioopt', # get optimal I/O size
|
||||
'--getra', # get readahead
|
||||
'--getalignoff', # get alignment offset in bytes
|
||||
'--getmaxsect', # get max sectors per request
|
||||
blockdev
|
||||
]
|
||||
opts = [o[5:] for o in cmd if o.startswith('--get')]
|
||||
report = utils.execute(*cmd, check_exit_code=[0])[0]
|
||||
return dict(zip(opts, report.splitlines()))
|
||||
|
||||
|
||||
def extrareport(dev):
|
||||
"""Builds device report using some additional sources.
|
||||
|
||||
:param dev: A device file, e.g. /dev/sda.
|
||||
|
||||
:returns: A dict of properties.
|
||||
"""
|
||||
spec = {}
|
||||
name = os.path.basename(dev)
|
||||
|
||||
# Finding out if block device is removable or not
|
||||
# actually, some disks are marked as removable
|
||||
# while they are actually not e.g. Adaptec RAID volumes
|
||||
try:
|
||||
with open('/sys/block/{0}/removable'.format(name)) as file:
|
||||
spec['removable'] = file.read().strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for key in ('state', 'timeout'):
|
||||
try:
|
||||
with open('/sys/block/{0}/device/{1}'.format(name, key)) as file:
|
||||
spec[key] = file.read().strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return spec
|
||||
|
||||
|
||||
def get_block_devices_from_udev_db():
|
||||
devs = []
|
||||
output = utils.execute('udevadm', 'info', '--export-db')[0]
|
||||
for device in output.split('\n\n'):
|
||||
# NOTE(agordeev): add only disks or their partitions
|
||||
if 'SUBSYSTEM=block' in device and ('DEVTYPE=disk' in device or
|
||||
'DEVTYPE=partition' in device):
|
||||
# NOTE(agordeev): it has to be sorted in order
|
||||
# to find MAJOR property prior DEVNAME property.
|
||||
for line in sorted(device.split('\n'), reverse=True):
|
||||
if line.startswith('E: MAJOR='):
|
||||
major = int(line.split()[1].split('=')[1])
|
||||
if major not in VALID_MAJORS:
|
||||
# NOTE(agordeev): filter out cd/dvd drives and other
|
||||
# block devices in which fuel-agent aren't interested
|
||||
break
|
||||
if line.startswith('E: DEVNAME='):
|
||||
d = line.split()[1].split('=')[1]
|
||||
if not any(os.path.basename(d).startswith(n)
|
||||
for n in ('nbd', 'ram', 'loop')):
|
||||
devs.append(line.split()[1].split('=')[1])
|
||||
break
|
||||
return devs
|
||||
|
||||
|
||||
def list_block_devices(disks=True):
|
||||
"""Gets list of block devices
|
||||
|
||||
Tries to guess which of them are disks
|
||||
and returns list of dicts representing those disks.
|
||||
|
||||
:returns: A list of dict representing disks available on a node.
|
||||
"""
|
||||
bdevs = []
|
||||
# NOTE(agordeev): blockdev from util-linux contains a bug
|
||||
# which's causing 'blockdev --report' to fail on reporting
|
||||
# nvme devices.
|
||||
# The actual fix is included in util-linux-2.24.1:
|
||||
# - don't use HDIO_GETGEO [Phillip Susi]
|
||||
# Since the bug only affects '--report' it is safe to use
|
||||
# 'blockdevreport'.
|
||||
# fuel-agent has to be switched to use udev database in order to
|
||||
# find all block devices recognized by kernel.
|
||||
devs = get_block_devices_from_udev_db()
|
||||
for device in devs:
|
||||
uspec = udevreport(device)
|
||||
espec = extrareport(device)
|
||||
# NOTE(agordeev): blockdevreport will fail if there's no medium
|
||||
# inserted into removable device.
|
||||
# Accept only devices from REMOVABLE_VENDORS list
|
||||
if (espec.get('removable') == '1' and
|
||||
uspec.get('ID_VENDOR') not in REMOVABLE_VENDORS):
|
||||
continue
|
||||
bspec = blockdevreport(device)
|
||||
|
||||
# if device is not disk,skip it
|
||||
if disks and not is_disk(device, bspec=bspec, uspec=uspec):
|
||||
continue
|
||||
|
||||
bdev = {
|
||||
'device': device,
|
||||
# NOTE(agordeev): blockdev gets 'startsec' from sysfs,
|
||||
# 'size' is determined by ioctl call.
|
||||
# This data was not actually used by fuel-agent,
|
||||
# so it can be removed without side effects.
|
||||
'uspec': uspec,
|
||||
'bspec': bspec,
|
||||
'espec': espec
|
||||
}
|
||||
bdevs.append(bdev)
|
||||
return bdevs
|
||||
|
||||
|
||||
def match_device(uspec1, uspec2):
|
||||
"""Tries to find out if uspec1 and uspec2 are uspecs from the same device
|
||||
|
||||
It compares only some fields in uspecs (not all of them) which, we believe,
|
||||
is enough to say exactly whether uspecs belong to the same device or not.
|
||||
|
||||
:param uspec1: A dict of properties which we get from udevadm.
|
||||
:param uspec1: A dict of properties which we get from udevadm.
|
||||
|
||||
:returns: True if uspecs match each other else False.
|
||||
"""
|
||||
|
||||
# False if ID_WWN is given and does not match each other
|
||||
if ('ID_WWN' in uspec1 and 'ID_WWN' in uspec2
|
||||
and uspec1['ID_WWN'] != uspec2['ID_WWN']):
|
||||
return False
|
||||
|
||||
# False if ID_SERIAL_SHORT is given and does not match each other
|
||||
if ('ID_SERIAL_SHORT' in uspec1 and 'ID_SERIAL_SHORT' in uspec2
|
||||
and uspec1['ID_SERIAL_SHORT'] != uspec2['ID_SERIAL_SHORT']):
|
||||
return False
|
||||
|
||||
# True if at least one by-id link is the same for both uspecs
|
||||
if ('DEVLINKS' in uspec1 and 'DEVLINKS' in uspec2
|
||||
and any(x.startswith('/dev/disk/by-id') for x in
|
||||
set(uspec1['DEVLINKS']) & set(uspec2['DEVLINKS']))):
|
||||
return True
|
||||
|
||||
# True if ID_WWN is given and matches each other
|
||||
# and DEVTYPE is given and is 'disk'
|
||||
if (uspec1.get('ID_WWN') == uspec2.get('ID_WWN') is not None
|
||||
and uspec1.get('DEVTYPE') == uspec2.get('DEVTYPE') == 'disk'):
|
||||
return True
|
||||
|
||||
# True if ID_WWN is given and matches each other
|
||||
# and DEVTYPE is given and is 'partition'
|
||||
# and MINOR is given and matches each other
|
||||
if (uspec1.get('ID_WWN') == uspec2.get('ID_WWN') is not None
|
||||
and uspec1.get('DEVTYPE') == uspec2.get('DEVTYPE') == 'partition'
|
||||
and uspec1.get('MINOR') == uspec2.get('MINOR') is not None):
|
||||
return True
|
||||
|
||||
# True if ID_SERIAL_SHORT is given and matches each other
|
||||
# and DEVTYPE is given and is 'disk'
|
||||
if (uspec1.get('ID_SERIAL_SHORT') == uspec2.get('ID_SERIAL_SHORT')
|
||||
is not None
|
||||
and uspec1.get('DEVTYPE') == uspec2.get('DEVTYPE') == 'disk'):
|
||||
return True
|
||||
|
||||
# True if DEVPATH is given and matches each other
|
||||
if uspec1.get('DEVPATH') == uspec2.get('DEVPATH') is not None:
|
||||
return True
|
||||
|
||||
return False
|
|
@ -1,243 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def pvdisplay():
|
||||
# unit m means MiB (power of 2)
|
||||
output = utils.execute(
|
||||
'pvdisplay',
|
||||
'-C',
|
||||
'--noheading',
|
||||
'--units', 'm',
|
||||
'--options', 'pv_name,vg_name,pv_size,dev_size,pv_uuid',
|
||||
'--separator', ';',
|
||||
check_exit_code=[0])[0]
|
||||
return pvdisplay_parse(output)
|
||||
|
||||
|
||||
def pvdisplay_parse(output):
|
||||
pvs = []
|
||||
for line in output.split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
pv_params = line.split(';')
|
||||
pvs.append({
|
||||
'name': pv_params[0],
|
||||
'vg': pv_params[1] or None,
|
||||
'psize': utils.parse_unit(pv_params[2], 'm'),
|
||||
'devsize': utils.parse_unit(pv_params[3], 'm'),
|
||||
'uuid': pv_params[4]
|
||||
})
|
||||
LOG.debug('Found physical volumes: {0}'.format(pvs))
|
||||
return pvs
|
||||
|
||||
|
||||
def pvcreate(pvname, metadatasize=64, metadatacopies=2):
|
||||
# check if pv already exists
|
||||
if filter(lambda x: x['name'] == pvname, pvdisplay()):
|
||||
raise errors.PVAlreadyExistsError(
|
||||
'Error while creating pv: pv %s already exists' % pvname)
|
||||
utils.execute('pvcreate',
|
||||
'--metadatacopies', str(metadatacopies),
|
||||
'--metadatasize', str(metadatasize) + 'm',
|
||||
pvname, check_exit_code=[0])
|
||||
|
||||
|
||||
def pvremove(pvname):
|
||||
pv = filter(lambda x: x['name'] == pvname, pvdisplay())
|
||||
|
||||
# check if pv exists
|
||||
if not pv:
|
||||
raise errors.PVNotFoundError(
|
||||
'Error while removing pv: pv %s not found' % pvname)
|
||||
# check if pv is attached to some vg
|
||||
if pv[0]['vg'] is not None:
|
||||
raise errors.PVBelongsToVGError('Error while removing pv: '
|
||||
'pv belongs to vg %s' % pv[0]['vg'])
|
||||
utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])
|
||||
|
||||
|
||||
def vgdisplay():
|
||||
output = utils.execute(
|
||||
'vgdisplay',
|
||||
'-C',
|
||||
'--noheading',
|
||||
'--units', 'm',
|
||||
'--options', 'vg_name,vg_uuid,vg_size,vg_free',
|
||||
'--separator', ';',
|
||||
check_exit_code=[0])[0]
|
||||
return vgdisplay_parse(output)
|
||||
|
||||
|
||||
def vgdisplay_parse(output):
|
||||
vgs = []
|
||||
for line in output.split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
vg_params = line.split(';')
|
||||
vgs.append({
|
||||
'name': vg_params[0],
|
||||
'uuid': vg_params[1],
|
||||
'size': utils.parse_unit(vg_params[2], 'm'),
|
||||
'free': utils.parse_unit(vg_params[3], 'm', ceil=False)
|
||||
})
|
||||
LOG.debug('Found volume groups: {0}'.format(vgs))
|
||||
return vgs
|
||||
|
||||
|
||||
def _vg_attach_validate(pvnames):
|
||||
pvs = pvdisplay()
|
||||
# check if all necessary pv exist
|
||||
if not set(pvnames).issubset(set([pv['name'] for pv in pvs])):
|
||||
raise errors.PVNotFoundError(
|
||||
'Error while creating vg: at least one of pv is not found')
|
||||
# check if all necessary pv are not already attached to some vg
|
||||
if not set(pvnames).issubset(
|
||||
set([pv['name'] for pv in pvs if pv['vg'] is None])):
|
||||
raise errors.PVBelongsToVGError(
|
||||
'Error while creating vg: at least one of pvs is '
|
||||
'already attached to some vg')
|
||||
|
||||
|
||||
def vgcreate(vgname, pvname, *args):
|
||||
# check if vg already exists
|
||||
if filter(lambda x: x['name'] == vgname, vgdisplay()):
|
||||
raise errors.VGAlreadyExistsError(
|
||||
'Error while creating vg: vg %s already exists' % vgname)
|
||||
pvnames = [pvname] + list(args)
|
||||
_vg_attach_validate(pvnames)
|
||||
utils.execute('vgcreate', vgname, *pvnames, check_exit_code=[0])
|
||||
|
||||
|
||||
def vgextend(vgname, pvname, *args):
|
||||
# check if vg exists
|
||||
if not filter(lambda x: x['name'] == vgname, vgdisplay()):
|
||||
raise errors.VGNotFoundError(
|
||||
'Error while extending vg: vg %s not found' % vgname)
|
||||
pvnames = [pvname] + list(args)
|
||||
_vg_attach_validate(pvnames)
|
||||
utils.execute('vgextend', vgname, *pvnames, check_exit_code=[0])
|
||||
|
||||
|
||||
def vgreduce(vgname, pvname, *args):
|
||||
# check if vg exists
|
||||
if not filter(lambda x: x['name'] == vgname, vgdisplay()):
|
||||
raise errors.VGNotFoundError(
|
||||
'Error while reducing vg: vg %s not found' % vgname)
|
||||
pvnames = [pvname] + list(args)
|
||||
# check if all necessary pv are attached to vg
|
||||
if not set(pvnames).issubset(
|
||||
set([pv['name'] for pv in pvdisplay() if pv['vg'] == vgname])):
|
||||
raise errors.PVNotFoundError(
|
||||
'Error while reducing vg: at least one of pv is '
|
||||
'not attached to vg')
|
||||
utils.execute('vgreduce', '-f', vgname, *pvnames, check_exit_code=[0])
|
||||
|
||||
|
||||
def vgremove(vgname):
|
||||
# check if vg exists
|
||||
if not filter(lambda x: x['name'] == vgname, vgdisplay()):
|
||||
raise errors.VGNotFoundError(
|
||||
'Error while removing vg: vg %s not found' % vgname)
|
||||
utils.execute('vgremove', '-f', vgname, check_exit_code=[0])
|
||||
|
||||
|
||||
def lvdisplay():
|
||||
output = utils.execute(
|
||||
'lvdisplay',
|
||||
'-C',
|
||||
'--noheading',
|
||||
'--units', 'm',
|
||||
# NOTE(agordeev): lv_path had been removed from options
|
||||
# since versions of lvdisplay prior 2.02.68 don't have it.
|
||||
'--options', 'lv_name,lv_size,vg_name,lv_uuid',
|
||||
'--separator', ';',
|
||||
check_exit_code=[0])[0]
|
||||
return lvdisplay_parse(output)
|
||||
|
||||
|
||||
def lvdisplay_parse(output):
|
||||
lvs = []
|
||||
for line in output.split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
lv_params = line.split(';')
|
||||
lvs.append({
|
||||
'name': lv_params[0],
|
||||
'size': utils.parse_unit(lv_params[1], 'm'),
|
||||
'vg': lv_params[2],
|
||||
'uuid': lv_params[3],
|
||||
# NOTE(agordeev): simulate lv_path with '/dev/$vg_name/$lv_name'
|
||||
'path': '/dev/%s/%s' % (lv_params[2], lv_params[0])
|
||||
})
|
||||
LOG.debug('Found logical volumes: {0}'.format(lvs))
|
||||
return lvs
|
||||
|
||||
|
||||
def lvcreate(vgname, lvname, size):
|
||||
vg = filter(lambda x: x['name'] == vgname, vgdisplay())
|
||||
|
||||
# check if vg exists
|
||||
if not vg:
|
||||
raise errors.VGNotFoundError(
|
||||
'Error while creating vg: vg %s not found' % vgname)
|
||||
# check if enough space is available
|
||||
if vg[0]['free'] < size:
|
||||
raise errors.NotEnoughSpaceError(
|
||||
'Error while creating lv: vg %s has only %s m of free space, '
|
||||
'but at least %s m is needed' % (vgname, vg[0]['free'], size))
|
||||
# check if lv already exists
|
||||
if filter(lambda x: x['name'] == lvname and x['vg'] == vgname,
|
||||
lvdisplay()):
|
||||
raise errors.LVAlreadyExistsError(
|
||||
'Error while creating lv: lv %s already exists' % lvname)
|
||||
# NOTE(agordeev): by default, lvcreate is configured to wipe signature
|
||||
# on allocated volume. '--yes' should be passed to avoid waiting for
|
||||
# user's confirmation:
|
||||
# "WARNING: <signature> signature detected on <device>. Wipe it? [y/n]"
|
||||
utils.execute('lvcreate', '--yes', '-L', '%sm' % size, '-n', lvname,
|
||||
vgname, check_exit_code=[0])
|
||||
|
||||
|
||||
def lvremove(lvpath):
|
||||
# check if lv exists
|
||||
if not filter(lambda x: x['path'] == lvpath, lvdisplay()):
|
||||
raise errors.LVNotFoundError(
|
||||
'Error while removing lv: lv %s not found' % lvpath)
|
||||
utils.execute('lvremove', '-f', lvpath, check_exit_code=[0])
|
||||
|
||||
|
||||
def lvremove_all():
|
||||
for lv in lvdisplay():
|
||||
lvremove(lv['path'])
|
||||
|
||||
|
||||
def vgremove_all():
|
||||
for vg in vgdisplay():
|
||||
vgremove(vg['name'])
|
||||
|
||||
|
||||
def pvremove_all():
|
||||
for pv in pvdisplay():
|
||||
pvremove(pv['name'])
|
|
@ -1,170 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import hardware as hu
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def mddetail_parse(output):
|
||||
md = {}
|
||||
# NOTE(agordeev): Somethimes 'State' column is missing
|
||||
h, _, v = re.split("Number\s+Major\s+Minor\s+RaidDevice\s+(State\s+)?",
|
||||
output)
|
||||
for line in h.split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
for pattern in ('Version', 'Raid Level', 'Raid Devices',
|
||||
'Active Devices', 'Spare Devices',
|
||||
'Failed Devices', 'State', 'UUID'):
|
||||
if line.startswith(pattern):
|
||||
md[pattern] = line.split()[-1]
|
||||
md['devices'] = []
|
||||
for line in v.split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
md['devices'].append(line.split()[-1])
|
||||
return md
|
||||
|
||||
|
||||
def get_mdnames(output=None):
|
||||
mdnames = []
|
||||
if not output:
|
||||
with open('/proc/mdstat') as f:
|
||||
output = f.read()
|
||||
for line in output.split('\n'):
|
||||
if line.startswith('md'):
|
||||
mdnames.append('/dev/%s' % line.split()[0])
|
||||
return mdnames
|
||||
|
||||
|
||||
def mddisplay(names=None):
|
||||
mdnames = names or get_mdnames()
|
||||
mds = []
|
||||
for mdname in mdnames:
|
||||
md = {'name': mdname}
|
||||
try:
|
||||
output = utils.execute('mdadm', '--detail', mdname,
|
||||
check_exit_code=[0])[0]
|
||||
LOG.debug('mdadm --detail %s output:\n%s', mdname, output)
|
||||
md.update(mddetail_parse(output))
|
||||
except errors.ProcessExecutionError as exc:
|
||||
LOG.debug(exc)
|
||||
continue
|
||||
finally:
|
||||
mds.append(md)
|
||||
LOG.debug('Found md devices: {0}'.format(mds))
|
||||
return mds
|
||||
|
||||
|
||||
def mdcreate(mdname, level, device, *args):
|
||||
mds = mddisplay()
|
||||
|
||||
# check if md device already exists
|
||||
if filter(lambda x: x['name'] == mdname, mds):
|
||||
raise errors.MDAlreadyExistsError(
|
||||
'Error while creating md: md %s already exists' % mdname)
|
||||
|
||||
# check if level argument is valid
|
||||
supported_levels = ('0', '1', 'raid0', 'raid1', 'stripe', 'mirror')
|
||||
if level not in supported_levels:
|
||||
raise errors.MDWrongSpecError(
|
||||
'Error while creating md device: '
|
||||
'level must be one of: %s' % ', '.join(supported_levels))
|
||||
|
||||
devices = [device] + list(args)
|
||||
|
||||
# check if all necessary devices exist
|
||||
if not set(devices).issubset(
|
||||
set([bd['device'] for bd in hu.list_block_devices(disks=False)])):
|
||||
raise errors.MDNotFoundError(
|
||||
'Error while creating md: at least one of devices is not found')
|
||||
|
||||
# check if devices are not parts of some md array
|
||||
if set(devices) & \
|
||||
set(reduce(lambda x, y: x + y,
|
||||
[md.get('devices', []) for md in mds], [])):
|
||||
raise errors.MDDeviceDuplicationError(
|
||||
'Error while creating md: at least one of devices is '
|
||||
'already in belongs to some md')
|
||||
|
||||
# FIXME: mdadm will ask user to continue creating if any device appears to
|
||||
# be a part of raid array. Superblock zeroing helps to avoid that.
|
||||
map(mdclean, devices)
|
||||
utils.execute('mdadm', '--create', '--force', mdname, '-e0.90',
|
||||
'--level=%s' % level,
|
||||
'--raid-devices=%s' % len(devices), *devices,
|
||||
check_exit_code=[0])
|
||||
|
||||
|
||||
def mdremove(mdname):
|
||||
# check if md exists
|
||||
if mdname not in get_mdnames():
|
||||
raise errors.MDNotFoundError(
|
||||
'Error while removing md: md %s not found' % mdname)
|
||||
# FIXME: The issue faced was quiet hard to reproduce and to figure out the
|
||||
# root cause. For unknown reason already removed md device is
|
||||
# unexpectedly returning back after a while from time to time making
|
||||
# new md device creation to fail.
|
||||
# Still the actual reason of its failure is unknown, but after a
|
||||
# searching on a web a mention was found about a race in udev
|
||||
# http://dev.bizo.com/2012/07/mdadm-device-or-resource-busy.html
|
||||
# The article recommends to disable udev's queue entirely during md
|
||||
# device manipulation which sounds rather unappropriate for our case.
|
||||
# And the link to original post on mailing list suggests to execute
|
||||
# `udevadm settle` before removing the md device.
|
||||
# here -> http://permalink.gmane.org/gmane.linux.raid/34027
|
||||
# So, what was done. `udevadm settle` calls were placed just
|
||||
# before any of `mdadm` calls and the analizyng the logs was started.
|
||||
# According to the manual `settle` is an option that "Watches the
|
||||
# udev event queue, and exits if all current events are handled".
|
||||
# That means it will wait for udev's finishing of processing the
|
||||
# events. According to the logs noticeable delay had been recognized
|
||||
# between `udevadm settle` and the next `mdadm` call.
|
||||
# The delay was about 150-200ms or even bigger. It was appeared
|
||||
# right before the `mdadm --stop` call. That just means that udev was
|
||||
# too busy with events when we start to modifiy md devices hard.
|
||||
# Thus `udevadm settle` is helping to avoid the later failure and
|
||||
# to prevent strange behaviour of md device.
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
|
||||
utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
|
||||
|
||||
|
||||
def mdclean(device):
|
||||
# we don't care if device actually exists or not
|
||||
utils.execute('mdadm', '--zero-superblock', '--force', device,
|
||||
check_exit_code=[0])
|
||||
|
||||
|
||||
def mdclean_all():
|
||||
LOG.debug('Trying to wipe out all md devices')
|
||||
for md in mddisplay():
|
||||
mdremove(md['name'])
|
||||
for dev in md.get('devices', []):
|
||||
mdclean(dev)
|
||||
# second attempt, remove stale inactive devices
|
||||
for md in mddisplay():
|
||||
mdremove(md['name'])
|
||||
mds = mddisplay()
|
||||
if len(mds) > 0:
|
||||
raise errors.MDRemovingError(
|
||||
'Error while removing mds: few devices still presented %s' % mds)
|
|
@ -1,194 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import time
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
from fuel_agent.utils import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_partition_info(output):
|
||||
lines = output.split('\n')
|
||||
generic_params = lines[1].rstrip(';').split(':')
|
||||
generic = {
|
||||
'dev': generic_params[0],
|
||||
'size': utils.parse_unit(generic_params[1], 'MiB'),
|
||||
'logical_block': int(generic_params[3]),
|
||||
'physical_block': int(generic_params[4]),
|
||||
'table': generic_params[5],
|
||||
'model': generic_params[6]
|
||||
}
|
||||
parts = []
|
||||
for line in lines[2:]:
|
||||
line = line.strip().rstrip(';')
|
||||
if not line:
|
||||
continue
|
||||
part_params = line.split(':')
|
||||
parts.append({
|
||||
'num': int(part_params[0]),
|
||||
'begin': utils.parse_unit(part_params[1], 'MiB'),
|
||||
'end': utils.parse_unit(part_params[2], 'MiB'),
|
||||
'size': utils.parse_unit(part_params[3], 'MiB'),
|
||||
'fstype': part_params[4] or None
|
||||
})
|
||||
return {'generic': generic, 'parts': parts}
|
||||
|
||||
|
||||
def info(dev):
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
output = utils.execute('parted', '-s', dev, '-m',
|
||||
'unit', 'MiB',
|
||||
'print', 'free',
|
||||
check_exit_code=[0])[0]
|
||||
LOG.debug('Info output: \n%s' % output)
|
||||
result = parse_partition_info(output)
|
||||
LOG.debug('Info result: %s' % result)
|
||||
return result
|
||||
|
||||
|
||||
def wipe(dev):
|
||||
# making an empty new table is equivalent to wiping the old one
|
||||
LOG.debug('Wiping partition table on %s (we assume it is equal '
|
||||
'to creating a new one)' % dev)
|
||||
make_label(dev)
|
||||
|
||||
|
||||
def make_label(dev, label='gpt'):
|
||||
"""Creates partition label on a device.
|
||||
|
||||
:param dev: A device file, e.g. /dev/sda.
|
||||
:param label: Partition label type 'gpt' or 'msdos'. Optional.
|
||||
|
||||
:returns: None
|
||||
"""
|
||||
LOG.debug('Trying to create %s partition table on device %s' %
|
||||
(label, dev))
|
||||
if label not in ('gpt', 'msdos'):
|
||||
raise errors.WrongPartitionLabelError(
|
||||
'Wrong partition label type: %s' % label)
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
out, err = utils.execute('parted', '-s', dev, 'mklabel', label,
|
||||
check_exit_code=[0, 1])
|
||||
LOG.debug('Parted output: \n%s' % out)
|
||||
reread_partitions(dev, out=out)
|
||||
|
||||
|
||||
def set_partition_flag(dev, num, flag, state='on'):
|
||||
"""Sets flag on a partition
|
||||
|
||||
:param dev: A device file, e.g. /dev/sda.
|
||||
:param num: Partition number
|
||||
:param flag: Flag name. Must be one of 'bios_grub', 'legacy_boot',
|
||||
'boot', 'raid', 'lvm'
|
||||
:param state: Desiable flag state. 'on' or 'off'. Default is 'on'.
|
||||
|
||||
:returns: None
|
||||
"""
|
||||
LOG.debug('Trying to set partition flag: dev=%s num=%s flag=%s state=%s' %
|
||||
(dev, num, flag, state))
|
||||
# parted supports more flags but we are interested in
|
||||
# setting only this subset of them.
|
||||
# not all of these flags are compatible with one another.
|
||||
if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm'):
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Unsupported partition flag: %s' % flag)
|
||||
if state not in ('on', 'off'):
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Wrong partition flag state: %s' % state)
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
out, err = utils.execute('parted', '-s', dev, 'set', str(num),
|
||||
flag, state, check_exit_code=[0, 1])
|
||||
LOG.debug('Parted output: \n%s' % out)
|
||||
reread_partitions(dev, out=out)
|
||||
|
||||
|
||||
def set_gpt_type(dev, num, type_guid):
|
||||
"""Sets guid on a partition.
|
||||
|
||||
:param dev: A device file, e.g. /dev/sda.
|
||||
:param num: Partition number
|
||||
:param type_guid: Partition type guid. Must be one of those listed
|
||||
on this page http://en.wikipedia.org/wiki/GUID_Partition_Table.
|
||||
This method does not check whether type_guid is valid or not.
|
||||
|
||||
:returns: None
|
||||
"""
|
||||
# TODO(kozhukalov): check whether type_guid is valid
|
||||
LOG.debug('Setting partition GUID: dev=%s num=%s guid=%s' %
|
||||
(dev, num, type_guid))
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
utils.execute('sgdisk', '--typecode=%s:%s' % (num, type_guid),
|
||||
dev, check_exit_code=[0])
|
||||
|
||||
|
||||
def make_partition(dev, begin, end, ptype):
|
||||
LOG.debug('Trying to create a partition: dev=%s begin=%s end=%s' %
|
||||
(dev, begin, end))
|
||||
if ptype not in ('primary', 'logical'):
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Wrong partition type: %s' % ptype)
|
||||
|
||||
# check begin >= end
|
||||
if begin >= end:
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Wrong boundaries: begin >= end')
|
||||
|
||||
# check if begin and end are inside one of free spaces available
|
||||
if not any(x['fstype'] == 'free' and begin >= x['begin'] and
|
||||
end <= x['end'] for x in info(dev)['parts']):
|
||||
raise errors.WrongPartitionSchemeError(
|
||||
'Invalid boundaries: begin and end '
|
||||
'are not inside available free space')
|
||||
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
out, err = utils.execute(
|
||||
'parted', '-a', 'optimal', '-s', dev, 'unit', 'MiB',
|
||||
'mkpart', ptype, str(begin), str(end), check_exit_code=[0, 1])
|
||||
LOG.debug('Parted output: \n%s' % out)
|
||||
reread_partitions(dev, out=out)
|
||||
|
||||
|
||||
def remove_partition(dev, num):
|
||||
LOG.debug('Trying to remove partition: dev=%s num=%s' % (dev, num))
|
||||
if not any(x['fstype'] != 'free' and x['num'] == num
|
||||
for x in info(dev)['parts']):
|
||||
raise errors.PartitionNotFoundError('Partition %s not found' % num)
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
||||
out, err = utils.execute('parted', '-s', dev, 'rm',
|
||||
str(num), check_exit_code=[0, 1])
|
||||
reread_partitions(dev, out=out)
|
||||
|
||||
|
||||
def reread_partitions(dev, out='Device or resource busy', timeout=60):
|
||||
# The reason for this method to exist is that old versions of parted
|
||||
# use ioctl(fd, BLKRRPART, NULL) to tell Linux to re-read partitions.
|
||||
# This system call does not work sometimes. So we try to re-read partition
|
||||
# table several times. Besides partprobe uses BLKPG instead, which
|
||||
# is better than BLKRRPART for this case. BLKRRPART tells Linux to re-read
|
||||
# partitions while BLKPG tells Linux which partitions are available
|
||||
# BLKPG is usually used as a fallback system call.
|
||||
begin = time.time()
|
||||
while 'Device or resource busy' in out:
|
||||
if time.time() > begin + timeout:
|
||||
raise errors.BaseError('Unable to re-read partition table on'
|
||||
'device %s' % dev)
|
||||
LOG.debug('Last time output contained "Device or resource busy". '
|
||||
'Trying to re-read partition table on device %s' % dev)
|
||||
time.sleep(2)
|
||||
out, err = utils.execute('partprobe', dev, check_exit_code=[0, 1])
|
||||
LOG.debug('Partprobe output: \n%s' % out)
|
||||
utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
|
|
@ -1,267 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import hashlib
|
||||
import locale
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import jinja2
|
||||
from oslo.config import cfg
|
||||
import requests
|
||||
import stevedore.driver
|
||||
import urllib3
|
||||
|
||||
from six.moves import zip_longest
|
||||
|
||||
from fuel_agent import errors
|
||||
from fuel_agent.openstack.common import log as logging
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
u_opts = [
|
||||
cfg.IntOpt(
|
||||
'http_max_retries',
|
||||
default=30,
|
||||
help='Maximum retries count for http requests. 0 means infinite',
|
||||
),
|
||||
cfg.FloatOpt(
|
||||
'http_request_timeout',
|
||||
# Setting it to 10 secs will allow fuel-agent to overcome the momentary
|
||||
# peak loads when network bandwidth becomes as low as 0.1MiB/s, thus
|
||||
# preventing of wasting too much retries on such false positives.
|
||||
default=10.0,
|
||||
help='Http request timeout in seconds',
|
||||
),
|
||||
cfg.FloatOpt(
|
||||
'http_retry_delay',
|
||||
default=2.0,
|
||||
help='Delay in seconds before the next http request retry',
|
||||
),
|
||||
cfg.IntOpt(
|
||||
'read_chunk_size',
|
||||
default=1048576,
|
||||
help='Block size of data to read for calculating checksum',
|
||||
),
|
||||
cfg.FloatOpt(
|
||||
'execute_retry_delay',
|
||||
default=2.0,
|
||||
help='Delay in seconds before the next exectuion will retry',
|
||||
),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(u_opts)
|
||||
|
||||
|
||||
# NOTE(agordeev): signature compatible with execute from oslo
|
||||
def execute(*cmd, **kwargs):
|
||||
command = ' '.join(cmd)
|
||||
LOG.debug('Trying to execute command: %s', command)
|
||||
commands = [c.strip() for c in re.split(ur'\|', command)]
|
||||
env = os.environ
|
||||
env['PATH'] = '/bin:/usr/bin:/sbin:/usr/sbin'
|
||||
env['LC_ALL'] = env['LANG'] = env['LANGUAGE'] = kwargs.pop('language', 'C')
|
||||
attempts = kwargs.pop('attempts', 1)
|
||||
check_exit_code = kwargs.pop('check_exit_code', [0])
|
||||
ignore_exit_code = False
|
||||
to_filename = kwargs.get('to_filename')
|
||||
cwd = kwargs.get('cwd')
|
||||
|
||||
if isinstance(check_exit_code, bool):
|
||||
ignore_exit_code = not check_exit_code
|
||||
check_exit_code = [0]
|
||||
elif isinstance(check_exit_code, int):
|
||||
check_exit_code = [check_exit_code]
|
||||
|
||||
to_file = None
|
||||
if to_filename:
|
||||
to_file = open(to_filename, 'wb')
|
||||
|
||||
for attempt in reversed(xrange(attempts)):
|
||||
try:
|
||||
process = []
|
||||
for c in commands:
|
||||
try:
|
||||
# NOTE(eli): Python's shlex implementation doesn't like
|
||||
# unicode. We have to convert to ascii before shlex'ing
|
||||
# the command. http://bugs.python.org/issue6988
|
||||
encoded_command = c.encode('ascii')
|
||||
|
||||
process.append(subprocess.Popen(
|
||||
shlex.split(encoded_command),
|
||||
env=env,
|
||||
stdin=(process[-1].stdout if process else None),
|
||||
stdout=(to_file
|
||||
if ((len(process) == len(commands) - 1) and
|
||||
to_file)
|
||||
else subprocess.PIPE),
|
||||
stderr=(subprocess.PIPE),
|
||||
cwd=cwd
|
||||
))
|
||||
except (OSError, ValueError) as e:
|
||||
raise errors.ProcessExecutionError(exit_code=1, stdout='',
|
||||
stderr=e, cmd=command)
|
||||
if len(process) >= 2:
|
||||
process[-2].stdout.close()
|
||||
stdout, stderr = process[-1].communicate()
|
||||
if (not ignore_exit_code and
|
||||
process[-1].returncode not in check_exit_code):
|
||||
raise errors.ProcessExecutionError(
|
||||
exit_code=process[-1].returncode, stdout=stdout,
|
||||
stderr=stderr, cmd=command)
|
||||
return (stdout, stderr)
|
||||
except errors.ProcessExecutionError as e:
|
||||
LOG.warning('Failed to execute command: %s', e)
|
||||
if not attempt:
|
||||
raise
|
||||
else:
|
||||
time.sleep(CONF.execute_retry_delay)
|
||||
|
||||
|
||||
def parse_unit(s, unit, ceil=True):
|
||||
"""Converts '123.1unit' string into ints
|
||||
|
||||
If ceil is True it will be rounded up (124)
|
||||
and and down (123) if ceil is False.
|
||||
"""
|
||||
|
||||
flt = locale.atof(s.split(unit)[0])
|
||||
if ceil:
|
||||
return int(math.ceil(flt))
|
||||
return int(math.floor(flt))
|
||||
|
||||
|
||||
def B2MiB(b, ceil=True):
|
||||
if ceil:
|
||||
return int(math.ceil(float(b) / 1024 / 1024))
|
||||
return int(math.floor(float(b) / 1024 / 1024))
|
||||
|
||||
|
||||
def get_driver(name):
|
||||
LOG.debug('Trying to get driver: fuel_agent.drivers.%s', name)
|
||||
driver = stevedore.driver.DriverManager(
|
||||
namespace='fuel_agent.drivers', name=name).driver
|
||||
LOG.debug('Found driver: %s', driver.__name__)
|
||||
return driver
|
||||
|
||||
|
||||
def render_and_save(tmpl_dir, tmpl_names, tmpl_data, file_name):
|
||||
env = jinja2.Environment(loader=jinja2.FileSystemLoader(tmpl_dir))
|
||||
template = env.get_or_select_template(tmpl_names)
|
||||
output = template.render(tmpl_data)
|
||||
try:
|
||||
with open(file_name, 'w') as f:
|
||||
f.write(output)
|
||||
except Exception:
|
||||
raise errors.TemplateWriteError(
|
||||
'Something goes wrong while trying to save'
|
||||
'templated data to {0}'.format(file_name))
|
||||
|
||||
|
||||
def calculate_md5(filename, size):
|
||||
hash = hashlib.md5()
|
||||
processed = 0
|
||||
with open(filename, "rb") as f:
|
||||
while processed < size:
|
||||
block = f.read(CONF.read_chunk_size)
|
||||
if block:
|
||||
block_len = len(block)
|
||||
if processed + block_len < size:
|
||||
hash.update(block)
|
||||
processed += block_len
|
||||
else:
|
||||
hash.update(block[:size - processed])
|
||||
break
|
||||
else:
|
||||
break
|
||||
return hash.hexdigest()
|
||||
|
||||
|
||||
def init_http_request(url, byte_range=0):
|
||||
LOG.debug('Trying to initialize http request object %s, byte range: %s'
|
||||
% (url, byte_range))
|
||||
retry = 0
|
||||
while True:
|
||||
if (CONF.http_max_retries == 0) or retry <= CONF.http_max_retries:
|
||||
try:
|
||||
response_obj = requests.get(
|
||||
url, stream=True,
|
||||
timeout=CONF.http_request_timeout,
|
||||
headers={'Range': 'bytes=%s-' % byte_range})
|
||||
except (socket.timeout,
|
||||
urllib3.exceptions.DecodeError,
|
||||
urllib3.exceptions.ProxyError,
|
||||
requests.exceptions.ConnectionError,
|
||||
requests.exceptions.Timeout,
|
||||
requests.exceptions.TooManyRedirects) as e:
|
||||
LOG.debug('Got non-critical error when accessing to %s '
|
||||
'on %s attempt: %s' % (url, retry + 1, e))
|
||||
else:
|
||||
LOG.debug('Successful http request to %s on %s retry' %
|
||||
(url, retry + 1))
|
||||
break
|
||||
retry += 1
|
||||
time.sleep(CONF.http_retry_delay)
|
||||
else:
|
||||
raise errors.HttpUrlConnectionError(
|
||||
'Exceeded maximum http request retries for %s' % url)
|
||||
response_obj.raise_for_status()
|
||||
return response_obj
|
||||
|
||||
|
||||
def makedirs_if_not_exists(path, mode=0o755):
|
||||
"""Create directory if it does not exist
|
||||
|
||||
:param path: Directory path
|
||||
:param mode: Directory mode (Default: 0o755)
|
||||
"""
|
||||
if not os.path.isdir(path):
|
||||
os.makedirs(path, mode=mode)
|
||||
|
||||
|
||||
def grouper(iterable, n, fillvalue=None):
|
||||
"""Collect data into fixed-length chunks or blocks"""
|
||||
args = [iter(iterable)] * n
|
||||
return zip_longest(*args, fillvalue=fillvalue)
|
||||
|
||||
|
||||
def guess_filename(path, regexp, sort=True, reverse=True):
|
||||
"""Tries to find a file by regexp in a given path.
|
||||
|
||||
This method is supposed to be mostly used for looking up
|
||||
for available kernel files which are usually 'vmlinuz-X.Y.Z-foo'.
|
||||
In order to find the newest one we can sort files in backward
|
||||
direction (by default).
|
||||
|
||||
:param path: Directory where to look for a file
|
||||
:param regexp: (String) Regular expression (must have python syntax)
|
||||
:param sort: (Bool) If True (by default), sort files before looking up.
|
||||
It can be necessary when regexp does not unambiguously correspond to file.
|
||||
:param reverse: (Bool) If True (by default), sort files
|
||||
in backward direction.
|
||||
"""
|
||||
filenames = os.listdir(path)
|
||||
if sort:
|
||||
filenames = sorted(filenames, reverse=reverse)
|
||||
for filename in filenames:
|
||||
if re.search(regexp, filename):
|
||||
return filename
|
||||
return None
|
|
@ -1,17 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pbr.version
|
||||
|
||||
version_info = pbr.version.VersionInfo('fuel-agent')
|
|
@ -1,10 +0,0 @@
|
|||
[DEFAULT]
|
||||
|
||||
# The list of modules to copy from oslo-incubator
|
||||
module=config.generator
|
||||
module=gettextutils
|
||||
module=log
|
||||
module=processutils
|
||||
|
||||
# The base module to hold the copy of openstack.common
|
||||
base=fuel_agent
|
|
@ -1,13 +0,0 @@
|
|||
Babel>=1.3
|
||||
eventlet>=0.13.0
|
||||
iso8601>=0.1.9
|
||||
jsonschema>=2.3.0
|
||||
oslo.config>=1.2.0
|
||||
oslo.serialization>=1.4.0
|
||||
six>=1.5.2
|
||||
pbr>=0.7.0
|
||||
Jinja2
|
||||
stevedore>=0.15
|
||||
requests>=1.2.3
|
||||
urllib3>=1.7
|
||||
PyYAML==3.10
|
|
@ -1,51 +0,0 @@
|
|||
[metadata]
|
||||
name = fuel-agent
|
||||
version = 7.0.0
|
||||
author = Mirantis
|
||||
author-email = fuel-dev@lists.launchpad.net
|
||||
summary = Fuel agent
|
||||
classifier =
|
||||
Development Status :: 4 - Beta
|
||||
Programming Language :: Python
|
||||
|
||||
[files]
|
||||
packages =
|
||||
fuel_agent
|
||||
|
||||
[entry_points]
|
||||
console_scripts =
|
||||
# TODO(kozhukalov): rename entry point
|
||||
provision = fuel_agent.cmd.agent:provision
|
||||
fa_partition = fuel_agent.cmd.agent:partition
|
||||
fa_configdrive = fuel_agent.cmd.agent:configdrive
|
||||
fa_copyimage = fuel_agent.cmd.agent:copyimage
|
||||
fa_bootloader = fuel_agent.cmd.agent:bootloader
|
||||
fa_build_image = fuel_agent.cmd.agent:build_image
|
||||
|
||||
fuel_agent.drivers =
|
||||
nailgun = fuel_agent.drivers.nailgun:Nailgun
|
||||
nailgun_build_image = fuel_agent.drivers.nailgun:NailgunBuildImage
|
||||
|
||||
[pbr]
|
||||
autodoc_index_modules = True
|
||||
# this variable is needed to avoid including files
|
||||
# from other subprojects in this repository
|
||||
skip_git_sdist = True
|
||||
|
||||
[global]
|
||||
setup-hooks =
|
||||
pbr.hooks.setup_hook
|
||||
fuel_agent.hooks.setup_hook
|
||||
|
||||
[build_sphinx]
|
||||
all_files = 1
|
||||
build-dir = doc/build
|
||||
source-dir = doc/source
|
||||
|
||||
[egg_info]
|
||||
tag_build =
|
||||
tag_date = 0
|
||||
tag_svn_revision = 0
|
||||
|
||||
[wheel]
|
||||
universal = 1
|
|
@ -1,20 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import setuptools
|
||||
|
||||
|
||||
setuptools.setup(
|
||||
setup_requires=['pbr'],
|
||||
pbr=True)
|
|
@ -1,6 +0,0 @@
|
|||
hacking>=0.8.0,<0.9
|
||||
mock==1.0.1
|
||||
oslotest==1.0
|
||||
testtools>=0.9.34
|
||||
pytest>=2.7.2
|
||||
pytest-cov>=1.8.1
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
PROJECT_NAME=${PROJECT_NAME:-fuel_agent}
|
||||
CFGFILE_NAME=${PROJECT_NAME}.conf.sample
|
||||
|
||||
if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
|
||||
CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
|
||||
elif [ -e etc/${CFGFILE_NAME} ]; then
|
||||
CFGFILE=etc/${CFGFILE_NAME}
|
||||
else
|
||||
echo "${0##*/}: can not find config file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
|
||||
trap "rm -rf $TEMPDIR" EXIT
|
||||
|
||||
tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
|
||||
|
||||
if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
|
||||
then
|
||||
echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
|
||||
echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
|
||||
exit 1
|
||||
fi
|
|
@ -1,126 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
print_hint() {
|
||||
echo "Try \`${0##*/} --help' for more information." >&2
|
||||
}
|
||||
|
||||
PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o:f: \
|
||||
--long help,base-dir:,package-name:,output-dir:,output-file:,module:,library: -- "$@")
|
||||
|
||||
if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
|
||||
|
||||
eval set -- "$PARSED_OPTIONS"
|
||||
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
echo "${0##*/} [options]"
|
||||
echo ""
|
||||
echo "options:"
|
||||
echo "-h, --help show brief help"
|
||||
echo "-b, --base-dir=DIR project base directory"
|
||||
echo "-p, --package-name=NAME project package name"
|
||||
echo "-o, --output-dir=DIR file output directory"
|
||||
echo "-f, --output-file=FILE file output directory"
|
||||
echo "-m, --module=MOD extra python module to interrogate for options"
|
||||
echo "-l, --library=LIB extra library that registers options for discovery"
|
||||
exit 0
|
||||
;;
|
||||
-b|--base-dir)
|
||||
shift
|
||||
BASEDIR=`echo $1 | sed -e 's/\/*$//g'`
|
||||
shift
|
||||
;;
|
||||
-p|--package-name)
|
||||
shift
|
||||
PACKAGENAME=`echo $1`
|
||||
shift
|
||||
;;
|
||||
-o|--output-dir)
|
||||
shift
|
||||
OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
|
||||
shift
|
||||
;;
|
||||
-f|--output-file)
|
||||
shift
|
||||
OUTPUTFILE=`echo $1 | sed -e 's/\/*$//g'`
|
||||
shift
|
||||
;;
|
||||
-m|--module)
|
||||
shift
|
||||
MODULES="$MODULES -m $1"
|
||||
shift
|
||||
;;
|
||||
-l|--library)
|
||||
shift
|
||||
LIBRARIES="$LIBRARIES -l $1"
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
BASEDIR=${BASEDIR:-`pwd`}
|
||||
if ! [ -d $BASEDIR ]
|
||||
then
|
||||
echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1
|
||||
elif [[ $BASEDIR != /* ]]
|
||||
then
|
||||
BASEDIR=$(cd "$BASEDIR" && pwd)
|
||||
fi
|
||||
|
||||
PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}}
|
||||
PACKAGENAME=`echo $PACKAGENAME | tr - _`
|
||||
TARGETDIR=$BASEDIR/$PACKAGENAME
|
||||
if ! [ -d $TARGETDIR ]
|
||||
then
|
||||
echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1
|
||||
fi
|
||||
|
||||
OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc}
|
||||
# NOTE(bnemec): Some projects put their sample config in etc/,
|
||||
# some in etc/$PACKAGENAME/
|
||||
if [ -d $OUTPUTDIR/$PACKAGENAME ]
|
||||
then
|
||||
OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME
|
||||
elif ! [ -d $OUTPUTDIR ]
|
||||
then
|
||||
echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'`
|
||||
find $TARGETDIR -type f -name "*.pyc" -delete
|
||||
FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" ! -path "*/nova/*" \
|
||||
-exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
|
||||
|
||||
RC_FILE="`dirname $0`/oslo.config.generator.rc"
|
||||
if test -r "$RC_FILE"
|
||||
then
|
||||
source "$RC_FILE"
|
||||
fi
|
||||
|
||||
for mod in ${FUEL_AGENT_CONFIG_GENERATOR_EXTRA_MODULES}; do
|
||||
MODULES="$MODULES -m $mod"
|
||||
done
|
||||
|
||||
for lib in ${FUEL_AGENT_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
|
||||
LIBRARIES="$LIBRARIES -l $lib"
|
||||
done
|
||||
|
||||
export EVENTLET_NO_GREENDNS=yes
|
||||
|
||||
OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
|
||||
[ "$OS_VARS" ] && eval "unset \$OS_VARS"
|
||||
DEFAULT_MODULEPATH=fuel_agent.openstack.common.config.generator
|
||||
MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
|
||||
OUTPUTFILE=${OUTPUTFILE:-$OUTPUTDIR/$PACKAGENAME.conf.sample}
|
||||
python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
|
||||
|
||||
# Hook to allow projects to append custom config file snippets
|
||||
CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)
|
||||
for CONCAT_FILE in $CONCAT_FILES; do
|
||||
cat $CONCAT_FILE >> $OUTPUTFILE
|
||||
done
|
|
@ -1,7 +0,0 @@
|
|||
#!/bin/bash
|
||||
tools_path=${tools_path:-$(dirname $0)}
|
||||
venv_path=${venv_path:-${tools_path}}
|
||||
venv_dir=${venv_name:-/../.venv}
|
||||
TOOLS=${tools_path}
|
||||
VENV=${venv:-${venv_path}/${venv_dir}}
|
||||
source ${VENV}/bin/activate && "$@"
|
|
@ -1,42 +0,0 @@
|
|||
[tox]
|
||||
minversion = 1.6
|
||||
skipsdist = True
|
||||
envlist = py26,py27,pep8
|
||||
|
||||
[testenv]
|
||||
usedevelop = True
|
||||
install_command = pip install --allow-external -U {opts} {packages}
|
||||
setenv = VIRTUAL_ENV={envdir}
|
||||
deps = -r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
commands =
|
||||
py.test -vv {posargs:fuel_agent/tests}
|
||||
|
||||
[tox:jenkins]
|
||||
downloadcache = ~/cache/pip
|
||||
|
||||
[testenv:pep8]
|
||||
deps = hacking==0.10.2
|
||||
commands =
|
||||
flake8 {posargs:fuel_agent}
|
||||
|
||||
[testenv:cover]
|
||||
setenv = VIRTUAL_ENV={envdir}
|
||||
commands =
|
||||
py.test --cov fuel_agent {posargs:fuel_agent/tests}
|
||||
|
||||
[testenv:venv]
|
||||
commands = {posargs:}
|
||||
|
||||
[testenv:devenv]
|
||||
envdir = devenv
|
||||
usedevelop = True
|
||||
|
||||
[flake8]
|
||||
exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,tools,docs
|
||||
show-pep8 = True
|
||||
show-source = True
|
||||
count = True
|
||||
|
||||
[hacking]
|
||||
import_exceptions = fuel_agent.openstack.common.gettextutils._,testtools.matchers
|
|
@ -1,5 +0,0 @@
|
|||
[DEFAULT]
|
||||
test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -s fuel_agent_ci/tests -p "*.py" $LISTOPT $IDOPTION
|
||||
test_id_option=--load-list $IDFILE
|
||||
test_list_option=--list
|
||||
test_run_concurrency=echo 1
|
|
@ -1,13 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
|
@ -1,13 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
|
@ -1,114 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
from fuel_agent_ci import manager as ci_manager
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-f', '--file', dest='env_file', action='store',
|
||||
type=str, help='Environment data file', required=True
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(dest='action')
|
||||
|
||||
env_parser = subparsers.add_parser('env')
|
||||
env_parser.add_argument(
|
||||
'-a', '--action', dest='env_action', action='store',
|
||||
type=str, help='Env action', required=True
|
||||
)
|
||||
env_parser.add_argument(
|
||||
'-k', '--kwargs', dest='env_kwargs', action='store',
|
||||
type=str, required=False,
|
||||
help='Env action kwargs, must be valid json or yaml',
|
||||
)
|
||||
env_parser.add_argument(
|
||||
'-K', '--kwargs_file', dest='env_kwargs_file', action='store',
|
||||
type=str, required=False,
|
||||
help='Env action kwargs file, content must be valid json or yaml',
|
||||
)
|
||||
|
||||
item_parser = subparsers.add_parser('item')
|
||||
item_parser.add_argument(
|
||||
'-t', '--type', dest='item_type', action='store',
|
||||
type=str, help='Item type', required=True
|
||||
)
|
||||
item_parser.add_argument(
|
||||
'-a', '--action', dest='item_action', action='store',
|
||||
type=str, help='Item action', required=True
|
||||
)
|
||||
item_parser.add_argument(
|
||||
'-n', '--name', dest='item_name', action='store',
|
||||
type=str, help='Item name', required=False
|
||||
)
|
||||
item_parser.add_argument(
|
||||
'-k', '--kwargs', dest='item_kwargs', action='store',
|
||||
type=str, required=False,
|
||||
help='Item action kwargs, must be valid json or yaml',
|
||||
)
|
||||
item_parser.add_argument(
|
||||
'-K', '--kwargs_file', dest='item_kwargs_file', action='store',
|
||||
type=str, required=False,
|
||||
help='Item action kwargs file, content must be valid json or yaml',
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
def term_handler(signum=None, sigframe=None):
|
||||
sys.exit()
|
||||
signal.signal(signal.SIGTERM, term_handler)
|
||||
signal.signal(signal.SIGINT, term_handler)
|
||||
|
||||
parser = parse_args()
|
||||
params, other_params = parser.parse_known_args()
|
||||
with open(params.env_file) as f:
|
||||
env_data = yaml.load(f.read())
|
||||
|
||||
manager = ci_manager.Manager(env_data)
|
||||
# print 'params: %s' % params
|
||||
# print 'other_params: %s' % other_params
|
||||
|
||||
if params.action == 'env':
|
||||
kwargs = {}
|
||||
if params.env_kwargs:
|
||||
kwargs.update(yaml.load(params.env_kwargs))
|
||||
elif params.env_kwargs_file:
|
||||
with open(params.env_kwargs_file) as f:
|
||||
kwargs.update(yaml.load(f.read()))
|
||||
manager.do_env(params.env_action, **kwargs)
|
||||
elif params.action == 'item':
|
||||
kwargs = {}
|
||||
if params.item_kwargs:
|
||||
kwargs.update(yaml.load(params.item_kwargs))
|
||||
elif params.item_kwargs_file:
|
||||
with open(params.item_kwargs_file) as f:
|
||||
kwargs.update(yaml.load(f.read()))
|
||||
manager.do_item(params.item_type, params.item_action,
|
||||
params.item_name, **kwargs)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
|
@ -1,66 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from fuel_agent_ci.drivers import common_driver
|
||||
from fuel_agent_ci.drivers import fabric_driver
|
||||
from fuel_agent_ci.drivers import libvirt_driver
|
||||
from fuel_agent_ci.drivers import pygit2_driver
|
||||
from fuel_agent_ci.drivers import simple_http_driver
|
||||
|
||||
|
||||
class Driver(object):
|
||||
default_hierarchy = {
|
||||
# these methods are from common_driver
|
||||
'artifact_get': common_driver,
|
||||
'artifact_clean': common_driver,
|
||||
'artifact_status': common_driver,
|
||||
|
||||
# these methods are from fabric_driver
|
||||
'ssh_status': fabric_driver,
|
||||
'ssh_put_content': fabric_driver,
|
||||
'ssh_put_file': fabric_driver,
|
||||
'ssh_get_file': fabric_driver,
|
||||
'ssh_run': fabric_driver,
|
||||
|
||||
# these methods are from libvirt_driver
|
||||
'net_start': libvirt_driver,
|
||||
'net_stop': libvirt_driver,
|
||||
'net_status': libvirt_driver,
|
||||
'vm_start': libvirt_driver,
|
||||
'vm_stop': libvirt_driver,
|
||||
'vm_status': libvirt_driver,
|
||||
'dhcp_start': libvirt_driver,
|
||||
'dhcp_stop': libvirt_driver,
|
||||
'dhcp_status': libvirt_driver,
|
||||
'tftp_start': libvirt_driver,
|
||||
'tftp_stop': libvirt_driver,
|
||||
'tftp_status': libvirt_driver,
|
||||
|
||||
# these methods are from pygit2_driver
|
||||
'repo_clone': pygit2_driver,
|
||||
'repo_clean': pygit2_driver,
|
||||
'repo_status': pygit2_driver,
|
||||
|
||||
# these methods are from simple_http_driver
|
||||
'http_start': simple_http_driver,
|
||||
'http_stop': simple_http_driver,
|
||||
'http_status': simple_http_driver,
|
||||
}
|
||||
|
||||
def __init__(self, hierarchy=None):
|
||||
self.hierarchy = self.default_hierarchy
|
||||
self.hierarchy.update(hierarchy or {})
|
||||
|
||||
def __getattr__(self, item):
|
||||
return getattr(self.hierarchy[item], item)
|
|
@ -1,127 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import requests
|
||||
|
||||
from fuel_agent_ci import utils
|
||||
|
||||
|
||||
def artifact_get(artifact):
|
||||
with open(os.path.join(artifact.env.envdir, artifact.path), 'wb') as f:
|
||||
for chunk in requests.get(
|
||||
artifact.url, stream=True).iter_content(1048576):
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
utils.execute(artifact.unpack, cwd=artifact.env.envdir)
|
||||
|
||||
|
||||
def artifact_clean(artifact):
|
||||
utils.execute(artifact.clean, cwd=artifact.env.envdir)
|
||||
|
||||
|
||||
def artifact_status(artifact):
|
||||
return os.path.isfile(os.path.join(artifact.env.envdir, artifact.path))
|
||||
|
||||
|
||||
def dhcp_start(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def dhcp_stop(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def dhcp_status(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def http_start(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def http_stop(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def http_status(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def net_start(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def net_stop(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def net_status(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def repo_clone(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def repo_clean(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def repo_status(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def ssh_status(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def ssh_put_content(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def ssh_put_file(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def ssh_get_file(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def ssh_run(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def tftp_start(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def tftp_stop(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def tftp_status(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def vm_start(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def vm_stop(*args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def vm_status(*args, **kwargs):
|
||||
raise NotImplementedError
|
|
@ -1,116 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from fabric import api as fab
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ssh_status(ssh):
|
||||
LOG.debug('Trying to get ssh status')
|
||||
with fab.settings(
|
||||
host_string=ssh.host,
|
||||
user=ssh.user,
|
||||
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
|
||||
timeout=ssh.connection_timeout):
|
||||
try:
|
||||
with fab.hide('running', 'stdout', 'stderr'):
|
||||
fab.run('echo')
|
||||
LOG.debug('Ssh connection is available')
|
||||
return True
|
||||
except SystemExit:
|
||||
sys.exit()
|
||||
except Exception:
|
||||
LOG.debug('Ssh connection is not available')
|
||||
return False
|
||||
|
||||
|
||||
def ssh_put_content(ssh, file_content, remote_filename):
|
||||
LOG.debug('Trying to put content into remote file: %s' % remote_filename)
|
||||
with fab.settings(
|
||||
host_string=ssh.host,
|
||||
user=ssh.user,
|
||||
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
|
||||
timeout=ssh.connection_timeout):
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
f.write(file_content)
|
||||
try:
|
||||
fab.put(f.file, remote_filename)
|
||||
except SystemExit:
|
||||
sys.exit()
|
||||
except Exception:
|
||||
LOG.error('Error while putting content into '
|
||||
'remote file: %s' % remote_filename)
|
||||
raise
|
||||
|
||||
|
||||
def ssh_put_file(ssh, filename, remote_filename):
|
||||
LOG.debug('Trying to put file on remote host: '
|
||||
'local=%s remote=%s' % (filename, remote_filename))
|
||||
with fab.settings(
|
||||
host_string=ssh.host,
|
||||
user=ssh.user,
|
||||
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
|
||||
timeout=ssh.connection_timeout):
|
||||
try:
|
||||
fab.put(filename, remote_filename)
|
||||
except SystemExit:
|
||||
sys.exit()
|
||||
except Exception:
|
||||
LOG.error('Error while putting file on remote host: '
|
||||
'local=%s remote=%s' % (filename, remote_filename))
|
||||
raise
|
||||
|
||||
|
||||
def ssh_get_file(ssh, remote_filename, filename):
|
||||
LOG.debug('Trying to get file from remote host: '
|
||||
'local=%s remote=%s' % (filename, remote_filename))
|
||||
with fab.settings(
|
||||
host_string=ssh.host,
|
||||
user=ssh.user,
|
||||
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
|
||||
timeout=ssh.connection_timeout):
|
||||
try:
|
||||
fab.get(remote_filename, filename)
|
||||
except SystemExit:
|
||||
sys.exit()
|
||||
except Exception:
|
||||
LOG.error('Error while getting file from remote host: '
|
||||
'local=%s remote=%s' % (filename, remote_filename))
|
||||
raise
|
||||
|
||||
|
||||
def ssh_run(ssh, command, command_timeout=10):
|
||||
LOG.debug('Trying to run command on remote host: %s' % command)
|
||||
with fab.settings(
|
||||
host_string=ssh.host,
|
||||
user=ssh.user,
|
||||
key_filename=os.path.join(ssh.env.envdir, ssh.key_filename),
|
||||
timeout=ssh.connection_timeout,
|
||||
command_timeout=command_timeout,
|
||||
warn_only=True):
|
||||
try:
|
||||
with fab.hide('running', 'stdout', 'stderr'):
|
||||
return fab.run(command, pty=True)
|
||||
except SystemExit:
|
||||
sys.exit()
|
||||
except Exception:
|
||||
LOG.error('Error while putting file on remote host: '
|
||||
'%s' % command)
|
||||
raise
|
|
@ -1,507 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import libvirt
|
||||
import xmlbuilder
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_file_size(path):
|
||||
with open(path, 'rb') as file:
|
||||
current = file.tell()
|
||||
try:
|
||||
file.seek(0, 2)
|
||||
size = file.tell()
|
||||
finally:
|
||||
file.seek(current)
|
||||
return size
|
||||
|
||||
|
||||
def get_qcow_size(path):
|
||||
p = subprocess.Popen(['qemu-img', 'info', path], stdout=subprocess.PIPE)
|
||||
output = p.communicate()[0]
|
||||
m = re.search(ur'.*?virtual size:.*?\((\d+) bytes\).*', output)
|
||||
return m.group(1)
|
||||
|
||||
|
||||
class LibvirtDriver(object):
|
||||
|
||||
def __init__(self, conn_str=None):
|
||||
self.conn = libvirt.open(conn_str or "qemu:///system")
|
||||
|
||||
def net_define(self, name, uuid=None, bridge_name=None,
|
||||
forward_mode=None, virtualport_type=None,
|
||||
ip_address=None, ip_netmask=None,
|
||||
dhcp=None, tftp_root=None):
|
||||
|
||||
xml = xmlbuilder.XMLBuilder('network')
|
||||
xml.name(name)
|
||||
|
||||
if uuid:
|
||||
xml.uuid(uuid)
|
||||
|
||||
if bridge_name:
|
||||
xml.bridge(name=bridge_name)
|
||||
|
||||
if forward_mode:
|
||||
xml.forward(mode=forward_mode)
|
||||
|
||||
if virtualport_type:
|
||||
xml.virtualport(type=virtualport_type)
|
||||
|
||||
if ip_address:
|
||||
with xml.ip(address=ip_address,
|
||||
netmask=(ip_netmask or '255.255.255.0')):
|
||||
|
||||
if tftp_root:
|
||||
xml.tftp(root=tftp_root)
|
||||
|
||||
if dhcp:
|
||||
with xml.dhcp:
|
||||
xml.range(start=dhcp['start'], end=dhcp['end'])
|
||||
if dhcp.get('hosts'):
|
||||
for host in dhcp['hosts']:
|
||||
kwargs = {'mac': host['mac'], 'ip': host['ip']}
|
||||
if host.get('name'):
|
||||
kwargs.update({'name': host['name']})
|
||||
xml.host(**kwargs)
|
||||
if dhcp.get('bootp'):
|
||||
if dhcp['bootp'].get('server'):
|
||||
xml.bootp(
|
||||
file=dhcp['bootp']['file'],
|
||||
server=dhcp['bootp']['server']
|
||||
)
|
||||
else:
|
||||
xml.bootp(file=dhcp['bootp']['file'])
|
||||
|
||||
net = self.conn.networkDefineXML(str(xml))
|
||||
return net.UUIDString()
|
||||
|
||||
def net_start(self, uuid):
|
||||
net = self.conn.networkLookupByUUIDString(uuid)
|
||||
net.create()
|
||||
|
||||
def net_destroy(self, uuid):
|
||||
net = self.conn.networkLookupByUUIDString(uuid)
|
||||
net.destroy()
|
||||
|
||||
def net_undefine(self, uuid):
|
||||
net = self.conn.networkLookupByUUIDString(uuid)
|
||||
net.undefine()
|
||||
|
||||
def net_uuid_by_name(self, name):
|
||||
net = self.conn.networkLookupByName(name)
|
||||
return net.UUIDString()
|
||||
|
||||
def net_list(self):
|
||||
return self.conn.listDefinedNetworks() + self.conn.listNetworks()
|
||||
|
||||
def net_list_active(self):
|
||||
return self.conn.listNetworks()
|
||||
|
||||
def net_list_notactive(self):
|
||||
return self.conn.listDefinedNetworks()
|
||||
|
||||
def net_status(self, uuid):
|
||||
return {
|
||||
0: 'notactive',
|
||||
1: 'running'
|
||||
}[self.conn.networkLookupByUUIDString(uuid).isActive()]
|
||||
|
||||
def _add_disk(self, xml, disk):
|
||||
with xml.disk(type='file', device='disk', cache='writeback'):
|
||||
xml.driver(name='qemu', type='qcow2')
|
||||
xml.source(file=disk['source_file'])
|
||||
xml.target(
|
||||
dev=disk['target_dev'], bus=disk.get('target_bus', 'scsi'))
|
||||
|
||||
def _add_interface(self, xml, interface):
|
||||
itype = interface.get('type', 'network')
|
||||
with xml.interface(type=itype):
|
||||
if itype == 'bridge':
|
||||
xml.source(bridge=interface['source_bridge'])
|
||||
elif itype == 'network':
|
||||
xml.source(network=interface['source_network'])
|
||||
xml.model(type=interface.get('model_type', 'e1000'))
|
||||
if interface.get('mac_address'):
|
||||
xml.mac(address=interface['mac_address'])
|
||||
if interface.get('virtualport_type'):
|
||||
xml.virtualport(type=interface['virtualport_type'])
|
||||
|
||||
def define(self, name, uuid=None, type='kvm', memory='2048', vcpu='1',
|
||||
arch='x86_64', boot=None, disks=None, interfaces=None):
|
||||
xml = xmlbuilder.XMLBuilder('domain', type=type)
|
||||
xml.name(name)
|
||||
if uuid:
|
||||
xml.uuid(uuid)
|
||||
|
||||
xml.memory(memory, unit='MiB')
|
||||
xml.vcpu(vcpu)
|
||||
|
||||
with xml.os:
|
||||
xml.type('hvm', arch=arch, machine='pc-1.0')
|
||||
if boot:
|
||||
if isinstance(boot, (list, tuple)):
|
||||
for dev in boot:
|
||||
xml.boot(dev=dev)
|
||||
elif isinstance(boot, (str, unicode)):
|
||||
xml.boot(dev=boot)
|
||||
xml.bootmenu(enable='no')
|
||||
|
||||
with xml.features:
|
||||
xml.acpi
|
||||
xml.apic
|
||||
xml.pae
|
||||
xml.clock(offset='utc')
|
||||
xml.on_poweroff('destroy')
|
||||
xml.on_reboot('restart')
|
||||
xml.on_crash('restart')
|
||||
|
||||
with xml.devices:
|
||||
if os.path.exists('/usr/bin/kvm'): # Debian
|
||||
xml.emulator('/usr/bin/kvm')
|
||||
elif os.path.exists('/usr/bin/qemu-kvm'): # Redhat
|
||||
xml.emulator('/usr/bin/qemu-kvm')
|
||||
|
||||
xml.input(type='mouse', bus='ps2')
|
||||
xml.graphics(type='vnc', port='-1', autoport='yes')
|
||||
with xml.video:
|
||||
xml.model(type='cirrus', vram='9216', heads='1')
|
||||
xml.address(type='pci', domain='0x0000',
|
||||
bus='0x00', slot='0x02', function='0x0')
|
||||
with xml.memballoon(model='virtio'):
|
||||
xml.address(type='pci', domain='0x0000',
|
||||
bus='0x00', slot='0x07', function='0x0')
|
||||
|
||||
if disks:
|
||||
if isinstance(disks, (list,)):
|
||||
for disk in disks:
|
||||
self._add_disk(xml, disk)
|
||||
else:
|
||||
self._add_disk(xml, disks)
|
||||
|
||||
if interfaces:
|
||||
if isinstance(interfaces, (list,)):
|
||||
for interface in interfaces:
|
||||
self._add_interface(xml, interface)
|
||||
else:
|
||||
self._add_interface(xml, interfaces)
|
||||
|
||||
dom = self.conn.defineXML(str(xml))
|
||||
return dom.UUIDString()
|
||||
|
||||
def destroy(self, uuid):
|
||||
dom = self.conn.lookupByUUIDString(uuid)
|
||||
dom.destroy()
|
||||
|
||||
def start(self, uuid):
|
||||
dom = self.conn.lookupByUUIDString(uuid)
|
||||
dom.create()
|
||||
|
||||
def undefine(self, uuid):
|
||||
dom = self.conn.lookupByUUIDString(uuid)
|
||||
dom.undefine()
|
||||
|
||||
def list(self):
|
||||
return (
|
||||
self.conn.listDefinedDomains() +
|
||||
[self.conn.lookupByID(dom).name()
|
||||
for dom in self.conn.listDomainsID()]
|
||||
)
|
||||
|
||||
def list_active(self):
|
||||
return [self.conn.lookupByID(dom).name()
|
||||
for dom in self.conn.listDomainsID()]
|
||||
|
||||
def list_notactive(self):
|
||||
return self.conn.listDefinedDomains()
|
||||
|
||||
def uuid_by_name(self, name):
|
||||
dom = self.conn.lookupByName(name)
|
||||
return dom.UUIDString()
|
||||
|
||||
def status(self, uuid):
|
||||
states = {
|
||||
libvirt.VIR_DOMAIN_NOSTATE: 'nostate',
|
||||
libvirt.VIR_DOMAIN_RUNNING: 'running',
|
||||
libvirt.VIR_DOMAIN_BLOCKED: 'blocked',
|
||||
libvirt.VIR_DOMAIN_PAUSED: 'paused',
|
||||
libvirt.VIR_DOMAIN_SHUTDOWN: 'shutdown',
|
||||
libvirt.VIR_DOMAIN_SHUTOFF: 'shutoff',
|
||||
libvirt.VIR_DOMAIN_CRASHED: 'crashed',
|
||||
libvirt.VIR_DOMAIN_PMSUSPENDED: 'suspended',
|
||||
}
|
||||
|
||||
dom = self.conn.lookupByUUIDString(uuid)
|
||||
return states.get(dom.state()[0], 'unknown')
|
||||
|
||||
def pool_define(self, name, path):
|
||||
xml = xmlbuilder.XMLBuilder('pool', type='dir')
|
||||
xml.name(name)
|
||||
with xml.target:
|
||||
xml.path(path)
|
||||
if not os.path.isdir(path):
|
||||
os.makedirs(path, 0o755)
|
||||
return self.conn.storagePoolCreateXML(str(xml)).UUIDString()
|
||||
|
||||
def pool_list(self):
|
||||
return (self.conn.listDefinedStoragePools() +
|
||||
self.conn.listStoragePools())
|
||||
|
||||
def pool_list_active(self):
|
||||
return self.conn.listStoragePools()
|
||||
|
||||
def pool_list_notactive(self):
|
||||
return self.conn.listDefinedStoragePools()
|
||||
|
||||
def pool_destroy(self, uuid):
|
||||
pool = self.conn.storagePoolLookupByUUIDString(uuid)
|
||||
pool.destroy()
|
||||
|
||||
def pool_start(self, uuid):
|
||||
pool = self.conn.storagePoolLookupByUUIDString(uuid)
|
||||
pool.create()
|
||||
|
||||
def pool_undefine(self, uuid):
|
||||
pool = self.conn.storagePoolLookupByUUIDString(uuid)
|
||||
pool.undefine()
|
||||
|
||||
def pool_uuid_by_name(self, name):
|
||||
pool = self.conn.storagePoolLookupByName(name)
|
||||
return pool.UUIDString()
|
||||
|
||||
def vol_create(self, name, capacity=None,
|
||||
base=None, pool_name='default',
|
||||
backing_store=False, base_plus=0):
|
||||
xml = xmlbuilder.XMLBuilder('volume')
|
||||
xml.name(name)
|
||||
xml.allocation('0', unit='MiB')
|
||||
if base:
|
||||
xml.capacity(str(int(get_qcow_size(base)) +
|
||||
int(base_plus) * 1048576))
|
||||
else:
|
||||
xml.capacity(capacity, unit='MiB')
|
||||
|
||||
with xml.target:
|
||||
xml.format(type='qcow2')
|
||||
|
||||
pool = self.conn.storagePoolLookupByName(pool_name)
|
||||
if base and backing_store:
|
||||
with xml.backingStore:
|
||||
xml.path(base)
|
||||
xml.format(type='qcow2')
|
||||
|
||||
vol = pool.createXML(str(xml), flags=0)
|
||||
|
||||
if base and not backing_store:
|
||||
self.volume_upload(vol.key(), base)
|
||||
|
||||
return vol.key()
|
||||
|
||||
def vol_list(self, pool_name='default'):
|
||||
pool = self.conn.storagePoolLookupByName(pool_name)
|
||||
return pool.listVolumes()
|
||||
|
||||
def vol_path(self, name, pool_name='default'):
|
||||
pool = self.conn.storagePoolLookupByName(pool_name)
|
||||
vol = pool.storageVolLookupByName(name)
|
||||
return vol.path()
|
||||
|
||||
def vol_delete(self, name, pool_name='default'):
|
||||
pool = self.conn.storagePoolLookupByName(pool_name)
|
||||
vol = pool.storageVolLookupByName(name)
|
||||
vol.delete(flags=0)
|
||||
|
||||
def chunk_render(self, stream, size, fd):
|
||||
return fd.read(size)
|
||||
|
||||
def volume_upload(self, name, path):
|
||||
size = get_file_size(path)
|
||||
with open(path, 'rb') as fd:
|
||||
stream = self.conn.newStream(0)
|
||||
self.conn.storageVolLookupByKey(name).upload(
|
||||
stream=stream, offset=0,
|
||||
length=size, flags=0)
|
||||
stream.sendAll(self.chunk_render, fd)
|
||||
stream.finish()
|
||||
|
||||
|
||||
def net_start(net, drv=None):
|
||||
if drv is None:
|
||||
drv = LibvirtDriver()
|
||||
LOG.debug('Starting network: %s' % net.name)
|
||||
netname = net.env.name + '_' + net.name
|
||||
net_kwargs = {
|
||||
'bridge_name': net.bridge,
|
||||
'forward_mode': 'nat',
|
||||
'ip_address': net.ip,
|
||||
}
|
||||
tftp = net.env.tftp_by_network(net.name)
|
||||
if tftp:
|
||||
net_kwargs['tftp_root'] = os.path.join(
|
||||
net.env.envdir, tftp.tftp_root)
|
||||
dhcp = net.env.dhcp_by_network(net.name)
|
||||
if dhcp:
|
||||
net_kwargs['dhcp'] = {
|
||||
'start': dhcp.begin,
|
||||
'end': dhcp.end,
|
||||
}
|
||||
if dhcp.bootp:
|
||||
net_kwargs['dhcp']['bootp'] = dhcp.bootp
|
||||
if dhcp.hosts:
|
||||
net_kwargs['dhcp']['hosts'] = dhcp.hosts
|
||||
drv.net_define(netname, **net_kwargs)
|
||||
drv.net_start(drv.net_uuid_by_name(netname))
|
||||
|
||||
|
||||
def net_stop(net, drv=None):
|
||||
if drv is None:
|
||||
drv = LibvirtDriver()
|
||||
LOG.debug('Stopping net: %s' % net.name)
|
||||
netname = net.env.name + '_' + net.name
|
||||
if netname in drv.net_list():
|
||||
uuid = drv.net_uuid_by_name(netname)
|
||||
if netname in drv.net_list_active():
|
||||
drv.net_destroy(uuid)
|
||||
drv.net_undefine(uuid)
|
||||
|
||||
|
||||
def net_status(net, drv=None):
|
||||
if drv is None:
|
||||
drv = LibvirtDriver()
|
||||
return (net.env.name + '_' + net.name in drv.net_list_active())
|
||||
|
||||
|
||||
def vm_start(vm, drv=None):
|
||||
if drv is None:
|
||||
drv = LibvirtDriver()
|
||||
LOG.debug('Starting vm: %s' % vm.name)
|
||||
vmname = vm.env.name + '_' + vm.name
|
||||
|
||||
if vm.env.name not in drv.pool_list():
|
||||
LOG.debug('Defining volume pool %s' % vm.env.name)
|
||||
drv.pool_define(vm.env.name, os.path.join(vm.env.envdir, 'volumepool'))
|
||||
if vm.env.name not in drv.pool_list_active():
|
||||
LOG.debug('Starting volume pool %s' % vm.env.name)
|
||||
drv.pool_start(drv.pool_uuid_by_name(vm.env.name))
|
||||
|
||||
disks = []
|
||||
for num, disk in enumerate(vm.disks):
|
||||
disk_name = vmname + '_%s' % num
|
||||
order = 'abcdefghijklmnopqrstuvwxyz'
|
||||
if disk_name not in drv.vol_list(pool_name=vm.env.name):
|
||||
if disk.base:
|
||||
LOG.debug('Creating vm disk: pool=%s vol=%s base=%s' %
|
||||
(vm.env.name, disk_name, disk.base))
|
||||
drv.vol_create(disk_name, base=disk.base,
|
||||
pool_name=vm.env.name)
|
||||
else:
|
||||
LOG.debug('Creating empty vm disk: pool=%s vol=%s '
|
||||
'capacity=%s' % (vm.env.name, disk_name, disk.size))
|
||||
drv.vol_create(disk_name, capacity=disk.size,
|
||||
pool_name=vm.env.name)
|
||||
disks.append({
|
||||
'source_file': drv.vol_path(disk_name, pool_name=vm.env.name),
|
||||
'target_dev': 'sd%s' % order[num],
|
||||
'target_bus': 'scsi',
|
||||
})
|
||||
|
||||
interfaces = []
|
||||
for interface in vm.interfaces:
|
||||
LOG.debug('Creating vm interface: net=%s mac=%s' %
|
||||
(vm.env.name + '_' + interface.network, interface.mac))
|
||||
interfaces.append({
|
||||
'type': 'network',
|
||||
'source_network': vm.env.name + '_' + interface.network,
|
||||
'mac_address': interface.mac,
|
||||
'model_type': interface.model_type
|
||||
})
|
||||
LOG.debug('Defining vm %s' % vm.name)
|
||||
drv.define(vmname, boot=vm.boot, disks=disks, interfaces=interfaces)
|
||||
LOG.debug('Starting vm %s' % vm.name)
|
||||
drv.start(drv.uuid_by_name(vmname))
|
||||
|
||||
|
||||
def vm_stop(vm, drv=None):
|
||||
if drv is None:
|
||||
drv = LibvirtDriver()
|
||||
LOG.debug('Stopping vm: %s' % vm.name)
|
||||
vmname = vm.env.name + '_' + vm.name
|
||||
if vmname in drv.list():
|
||||
uuid = drv.uuid_by_name(vmname)
|
||||
if vmname in drv.list_active():
|
||||
LOG.debug('Destroying vm: %s' % vm.name)
|
||||
drv.destroy(uuid)
|
||||
LOG.debug('Undefining vm: %s' % vm.name)
|
||||
drv.undefine(uuid)
|
||||
|
||||
for volname in [v for v in drv.vol_list(pool_name=vm.env.name)
|
||||
if v.startswith(vmname)]:
|
||||
LOG.debug('Deleting vm disk: pool=%s vol=%s' % (vm.env.name, volname))
|
||||
drv.vol_delete(volname, pool_name=vm.env.name)
|
||||
|
||||
if not drv.vol_list(pool_name=vm.env.name):
|
||||
LOG.debug('Deleting volume pool: %s' % vm.env.name)
|
||||
if vm.env.name in drv.pool_list():
|
||||
uuid = drv.pool_uuid_by_name(vm.env.name)
|
||||
if vm.env.name in drv.pool_list_active():
|
||||
LOG.debug('Destroying pool: %s' % vm.env.name)
|
||||
drv.pool_destroy(uuid)
|
||||
if vm.env.name in drv.pool_list():
|
||||
LOG.debug('Undefining pool: %s' % vm.env.name)
|
||||
drv.pool_undefine(uuid)
|
||||
|
||||
|
||||
def vm_status(vm, drv=None):
|
||||
if drv is None:
|
||||
drv = LibvirtDriver()
|
||||
return (vm.env.name + '_' + vm.name in drv.list_active())
|
||||
|
||||
|
||||
def dhcp_start(dhcp):
|
||||
"""This feature is implemented in net_start
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def dhcp_stop(dhcp):
|
||||
"""This feature is implemented is net_stop
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def dhcp_status(dhcp):
|
||||
return dhcp.env.net_by_name(dhcp.network).status()
|
||||
|
||||
|
||||
def tftp_start(tftp):
|
||||
"""This feature is implemented is net_start
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def tftp_stop(tftp):
|
||||
"""This feature is implemented is net_stop
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def tftp_status(tftp):
|
||||
return tftp.env.net_by_name(tftp.network).status()
|
|
@ -1,37 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
import pygit2
|
||||
|
||||
from fuel_agent_ci import utils
|
||||
|
||||
|
||||
def repo_clone(repo):
|
||||
return pygit2.clone_repository(
|
||||
repo.url, os.path.join(repo.env.envdir, repo.path),
|
||||
checkout_branch=repo.branch)
|
||||
|
||||
|
||||
def repo_clean(repo):
|
||||
utils.execute('rm -rf %s' % os.path.join(repo.env.envdir, repo.path))
|
||||
|
||||
|
||||
def repo_status(repo):
|
||||
try:
|
||||
pygit2.discover_repository(os.path.join(repo.env.envdir, repo.path))
|
||||
except KeyError:
|
||||
return False
|
||||
return True
|
|
@ -1,235 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import atexit
|
||||
import BaseHTTPServer
|
||||
import errno
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import SimpleHTTPServer
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Cwd(object):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
self.orig_path = os.getcwd()
|
||||
|
||||
def __enter__(self):
|
||||
os.chdir(self.path)
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
os.chdir(self.orig_path)
|
||||
|
||||
|
||||
class CustomHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path == self.server.parent.shutdown_url:
|
||||
LOG.info('Shutdown request has been received: %s' % (self.path))
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.server.parent.stop_self()
|
||||
elif self.path == self.server.parent.status_url:
|
||||
LOG.info('Status request has been received: %s' % (self.path))
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
else:
|
||||
with Cwd(self.server.parent.rootpath):
|
||||
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
|
||||
|
||||
def do_HEAD(self):
|
||||
with Cwd(self.server.parent.rootpath):
|
||||
SimpleHTTPServer.SimpleHTTPRequestHandler.do_HEAD(self)
|
||||
|
||||
|
||||
class CustomHTTPServer(object):
|
||||
def __init__(self, host, port, rootpath,
|
||||
shutdown_url='/shutdown',
|
||||
status_url='/status',
|
||||
piddir='/var/run',
|
||||
pidfile='custom_httpd.pid',
|
||||
stdin=None, stdout=None, stderr=None):
|
||||
|
||||
self.host = str(host)
|
||||
self.port = int(port)
|
||||
|
||||
self.rootpath = rootpath
|
||||
self.shutdown_url = shutdown_url
|
||||
self.status_url = status_url
|
||||
self.stdin = stdin
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
self.pidfile = os.path.join(piddir, pidfile)
|
||||
|
||||
# We cannot just inherit BaseHTTPServer.HTTPServer because
|
||||
# it tries to bind socket during initialization but we need it
|
||||
# to be done during actual launching.
|
||||
self.server = None
|
||||
|
||||
def stop_self(self):
|
||||
if self.server:
|
||||
# We cannot use server.shutdown() here because
|
||||
# it sets _BaseServer__shutdown_request to True
|
||||
# end wait for _BaseServer__is_shut_down event to be set
|
||||
# that locks thread forever. We can use shutdown() method
|
||||
# from outside this thread.
|
||||
self.server._BaseServer__shutdown_request = True
|
||||
|
||||
def daemonize(self):
|
||||
# in order to avoid http process
|
||||
# to become zombie we need to fork twice
|
||||
try:
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
sys.exit(0)
|
||||
except OSError as e:
|
||||
sys.stderr.write('Error while fork#1 HTTP server: '
|
||||
'%d (%s)' % (e.errno, e.strerror))
|
||||
sys.exit(1)
|
||||
|
||||
os.chdir('/')
|
||||
os.setsid()
|
||||
os.umask(0)
|
||||
|
||||
try:
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
sys.exit(0)
|
||||
except OSError as e:
|
||||
sys.stderr.write('Error while fork#2 HTTP server: '
|
||||
'%d (%s)' % (e.errno, e.strerror))
|
||||
sys.exit(1)
|
||||
|
||||
if self.stdin:
|
||||
si = file(self.stdin, 'r')
|
||||
os.dup2(si.fileno(), sys.stdin.fileno())
|
||||
if self.stdout:
|
||||
sys.stdout.flush()
|
||||
so = file(self.stdout, 'a+')
|
||||
os.dup2(so.fileno(), sys.stdout.fileno())
|
||||
if self.stderr:
|
||||
sys.stderr.flush()
|
||||
se = file(self.stderr, 'a+', 0)
|
||||
os.dup2(se.fileno(), sys.stderr.fileno())
|
||||
|
||||
atexit.register(self.delpid)
|
||||
pid = str(os.getpid())
|
||||
with open(self.pidfile, 'w+') as f:
|
||||
f.write('%s\n' % pid)
|
||||
f.flush()
|
||||
|
||||
def delpid(self):
|
||||
os.remove(self.pidfile)
|
||||
|
||||
def run(self):
|
||||
self.server = BaseHTTPServer.HTTPServer(
|
||||
(self.host, self.port), CustomHTTPRequestHandler)
|
||||
self.server.parent = self
|
||||
self.server.serve_forever()
|
||||
|
||||
def check_pid(self, pid):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError as err:
|
||||
if err.errno == errno.ESRCH:
|
||||
# ESRCH == No such process
|
||||
return False
|
||||
elif err.errno == errno.EPERM:
|
||||
# EPERM clearly means there's a process to deny access to
|
||||
return True
|
||||
else:
|
||||
# According to "man 2 kill" possible error values are
|
||||
# (EINVAL, EPERM, ESRCH)
|
||||
return True
|
||||
|
||||
def start(self):
|
||||
try:
|
||||
with open(self.pidfile) as f:
|
||||
pid = int(f.read().strip())
|
||||
except (IOError, ValueError):
|
||||
pid = None
|
||||
if pid:
|
||||
if self.check_pid(pid):
|
||||
message = 'pidfile %s already exists. Daemon already running?\n'
|
||||
sys.stderr.write(message % self.pidfile)
|
||||
sys.exit(1)
|
||||
else:
|
||||
message = "pidfile %s already exists. Daemon isn't running. Removing the pidfile\n"
|
||||
sys.stderr.write(message % self.pidfile)
|
||||
self.delpid()
|
||||
self.daemonize()
|
||||
self.run()
|
||||
|
||||
def stop(self):
|
||||
try:
|
||||
with open(self.pidfile) as f:
|
||||
pid = int(f.read().strip())
|
||||
except (IOError, ValueError):
|
||||
pid = None
|
||||
if not pid:
|
||||
message = 'pidfile %s does not exist. Daemon not running?\n'
|
||||
sys.stderr.write(message % self.pidfile)
|
||||
return
|
||||
try:
|
||||
while True:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
time.sleep(1)
|
||||
except OSError as err:
|
||||
err = str(err)
|
||||
if err.find('No such process') > 0:
|
||||
if os.path.exists(self.pidfile):
|
||||
os.remove(self.pidfile)
|
||||
else:
|
||||
sys.stdout.write(str(err))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def http_start(http):
|
||||
def start():
|
||||
server = CustomHTTPServer(
|
||||
http.env.net_by_name(http.network).ip, http.port,
|
||||
os.path.join(http.env.envdir, http.http_root),
|
||||
status_url=http.status_url, shutdown_url=http.shutdown_url,
|
||||
pidfile=os.path.join(http.env.envdir,
|
||||
http.env.name + '_custom_httpd.pid'))
|
||||
server.start()
|
||||
multiprocessing.Process(target=start).start()
|
||||
|
||||
|
||||
def http_stop(http):
|
||||
if http_status(http):
|
||||
requests.get(
|
||||
'http://%s:%s%s' % (http.env.net_by_name(http.network).ip,
|
||||
http.port, http.shutdown_url))
|
||||
|
||||
|
||||
def http_status(http):
|
||||
try:
|
||||
status = requests.get(
|
||||
'http://%s:%s%s' % (http.env.net_by_name(http.network).ip,
|
||||
http.port, http.status_url),
|
||||
timeout=1
|
||||
)
|
||||
if status.status_code == 200:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
|
@ -1,31 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects.environment import Environment
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, data):
|
||||
self.env = Environment.new(**data)
|
||||
|
||||
def do_item(self, item_type, item_action, item_name=None, **kwargs):
|
||||
return getattr(
|
||||
self.env, '%s_%s' % (item_type, item_action))(item_name, **kwargs)
|
||||
|
||||
def do_env(self, env_action, **kwargs):
|
||||
return getattr(self.env, env_action)(**kwargs)
|
|
@ -1,34 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This mapping is supposed to be dynamically filled with
|
||||
# names of objects and their types
|
||||
|
||||
OBJECT_TYPES = {}
|
||||
|
||||
|
||||
class MetaObject(type):
|
||||
def __init__(self, name, bases, dct):
|
||||
if '__typename__' in dct:
|
||||
OBJECT_TYPES[dct['__typename__']] = self
|
||||
return super(MetaObject, self).__init__(name, bases, dct)
|
||||
|
||||
|
||||
class Object(object):
|
||||
__metaclass__ = MetaObject
|
||||
__typename__ = 'object'
|
||||
|
||||
@property
|
||||
def typename(self):
|
||||
return self.__typename__
|
|
@ -1,46 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Artifact(Object):
|
||||
__typename__ = 'artifact'
|
||||
|
||||
def __init__(self, env, name, url, path, unpack=None, clean=None):
|
||||
self.env = env
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.path = path
|
||||
self.unpack = unpack
|
||||
self.clean = clean
|
||||
|
||||
def get(self):
|
||||
if not self.status():
|
||||
LOG.debug('Getting artifact %s' % self.name)
|
||||
self.env.driver.artifact_get(self)
|
||||
|
||||
def clean(self):
|
||||
if self.status():
|
||||
LOG.debug('Cleaning artifact %s' % self.name)
|
||||
self.env.driver.artifact_clean(self)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.artifact_status(self)
|
||||
LOG.debug('Artifact %s status %s' % (self.name, status))
|
||||
return status
|
|
@ -1,56 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Dhcp(Object):
|
||||
__typename__ = 'dhcp'
|
||||
|
||||
def __init__(self, env, name, begin, end, network):
|
||||
self.name = name
|
||||
self.env = env
|
||||
self.begin = begin
|
||||
self.end = end
|
||||
self.network = network
|
||||
self.hosts = []
|
||||
self.bootp = None
|
||||
|
||||
def add_host(self, mac, ip, name=None):
|
||||
host = {'mac': mac, 'ip': ip}
|
||||
if name is not None:
|
||||
host['name'] = name
|
||||
self.hosts.append(host)
|
||||
|
||||
def set_bootp(self, file):
|
||||
self.bootp = {'file': file}
|
||||
|
||||
def start(self):
|
||||
if not self.status():
|
||||
LOG.debug('Starting DHCP')
|
||||
self.env.driver.dhcp_start(self)
|
||||
|
||||
def stop(self):
|
||||
if self.status():
|
||||
LOG.debug('Stopping DHCP')
|
||||
self.env.driver.dhcp_stop(self)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.dhcp_status(self)
|
||||
LOG.debug('DHCP status %s' % status)
|
||||
return status
|
|
@ -1,181 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from fuel_agent_ci import drivers
|
||||
from fuel_agent_ci.objects.artifact import Artifact
|
||||
from fuel_agent_ci.objects.dhcp import Dhcp
|
||||
from fuel_agent_ci.objects.http import Http
|
||||
from fuel_agent_ci.objects.net import Net
|
||||
from fuel_agent_ci.objects import OBJECT_TYPES
|
||||
from fuel_agent_ci.objects.repo import Repo
|
||||
from fuel_agent_ci.objects.ssh import Ssh
|
||||
from fuel_agent_ci.objects.tftp import Tftp
|
||||
from fuel_agent_ci.objects.vm import Vm
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Environment(object):
|
||||
def __init__(self, name, envdir, driver=None):
|
||||
self.name = name
|
||||
self.envdir = envdir
|
||||
self.driver = driver or drivers.Driver()
|
||||
self.items = []
|
||||
|
||||
@classmethod
|
||||
def new(cls, **kwargs):
|
||||
LOG.debug('Creating environment: %s' % kwargs['name'])
|
||||
envdir = kwargs.get('envdir') or os.path.join(
|
||||
tempfile.gettempdir(), kwargs['name'])
|
||||
if not os.path.exists(envdir):
|
||||
LOG.debug('Envdir %s does not exist. Creating envdir.' % envdir)
|
||||
os.makedirs(envdir)
|
||||
env = cls(kwargs['name'], envdir)
|
||||
for item_type in OBJECT_TYPES.keys():
|
||||
for item_kwargs in kwargs.get(item_type, []):
|
||||
LOG.debug('Creating %s: %s' % (item_type, item_kwargs))
|
||||
getattr(env, '%s_add' % item_type)(**item_kwargs)
|
||||
return env
|
||||
|
||||
def __getattr__(self, attr_name):
|
||||
"""This method maps item_add, item_by_name, item_action attributes into
|
||||
attributes for particular types like artifact_add or dhcp_by_name.
|
||||
|
||||
:param attr_name: Attribute name to map (e.g. net_add, repo_clone)
|
||||
|
||||
:returns: Lambda which implements a particular attribute.
|
||||
"""
|
||||
try:
|
||||
item_type, item_action = attr_name.split('_', 1)
|
||||
except Exception:
|
||||
raise AttributeError('Attribute %s not found' % attr_name)
|
||||
else:
|
||||
if item_action == 'add':
|
||||
return functools.partial(self.item_add, item_type)
|
||||
elif item_action == 'by_name':
|
||||
return functools.partial(self.item_by_name, item_type)
|
||||
else:
|
||||
return functools.partial(self.item_action,
|
||||
item_type, item_action)
|
||||
|
||||
def item_add(self, item_type, **kwargs):
|
||||
if self.item_by_name(item_type, kwargs.get('name')):
|
||||
raise Exception('Error while adding item: %s %s already exist' %
|
||||
(item_type, kwargs.get('name')))
|
||||
item = OBJECT_TYPES[item_type](env=self, **kwargs)
|
||||
self.items.append(item)
|
||||
return item
|
||||
|
||||
def vm_add(self, **kwargs):
|
||||
if self.item_by_name('vm', kwargs.get('name')):
|
||||
raise Exception('Error while adding vm: vm %s already exist' %
|
||||
kwargs.get('name'))
|
||||
disks = kwargs.pop('disks', [])
|
||||
interfaces = kwargs.pop('interfaces', [])
|
||||
vm = Vm(env=self, **kwargs)
|
||||
for disk_kwargs in disks:
|
||||
vm.add_disk(**disk_kwargs)
|
||||
for interface_kwargs in interfaces:
|
||||
vm.add_interface(**interface_kwargs)
|
||||
self.items.append(vm)
|
||||
return vm
|
||||
|
||||
def dhcp_add(self, **kwargs):
|
||||
if self.item_by_name('dhcp', kwargs.get('name')):
|
||||
raise Exception('Error while adding dhcp: dhcp %s already exist' %
|
||||
kwargs.get('name'))
|
||||
hosts = kwargs.pop('hosts', [])
|
||||
bootp_kwargs = kwargs.pop('bootp', None)
|
||||
dhcp = Dhcp(env=self, **kwargs)
|
||||
for host_kwargs in hosts:
|
||||
dhcp.add_host(**host_kwargs)
|
||||
if bootp_kwargs is not None:
|
||||
dhcp.set_bootp(**bootp_kwargs)
|
||||
self.items.append(dhcp)
|
||||
return dhcp
|
||||
|
||||
def item_by_name(self, item_type, item_name):
|
||||
found = filter(
|
||||
lambda x: x.typename == item_type and x.name == item_name,
|
||||
self.items
|
||||
)
|
||||
if not found or len(found) > 1:
|
||||
LOG.debug('Item %s %s not found' % (item_type, item_name))
|
||||
return None
|
||||
return found[0]
|
||||
|
||||
def item_action(self, item_type, item_action, item_name=None, **kwargs):
|
||||
if item_name:
|
||||
item = self.item_by_name(item_type, item_name)
|
||||
return {item_name: getattr(item, item_action)(**kwargs)}
|
||||
else:
|
||||
result = {}
|
||||
for item in [i for i in self.items if i.typename == item_type]:
|
||||
LOG.debug('Trying to do action on item: '
|
||||
'type=%s name=%s action=%s' %
|
||||
(item_type, item.name, item_action))
|
||||
result[item.name] = getattr(item, item_action)(**kwargs)
|
||||
return result
|
||||
|
||||
# TODO(kozhukalov): implement this method as classmethod in tftp object
|
||||
def tftp_by_network(self, network):
|
||||
found = filter(
|
||||
lambda x: x.typename == 'tftp' and x.network == network,
|
||||
self.items
|
||||
)
|
||||
if not found or len(found) > 1:
|
||||
LOG.debug('Tftp not found')
|
||||
return None
|
||||
return found[0]
|
||||
|
||||
# TODO(kozhukalov): implement this method as classmethod in dhcp object
|
||||
def dhcp_by_network(self, network):
|
||||
found = filter(
|
||||
lambda x: x.typename == 'dhcp' and x.network == network,
|
||||
self.items
|
||||
)
|
||||
if not found or len(found) > 1:
|
||||
LOG.debug('Dhcp not found')
|
||||
return None
|
||||
return found[0]
|
||||
|
||||
def start(self):
|
||||
LOG.debug('Starting environment')
|
||||
self.artifact_get()
|
||||
self.repo_clone()
|
||||
self.net_start()
|
||||
self.tftp_start()
|
||||
self.dhcp_start()
|
||||
self.http_start()
|
||||
self.vm_start()
|
||||
|
||||
def stop(self, artifact_clean=False, repo_clean=False):
|
||||
LOG.debug('Stopping environment')
|
||||
self.vm_stop()
|
||||
self.tftp_stop()
|
||||
self.dhcp_stop()
|
||||
self.http_stop()
|
||||
self.net_stop()
|
||||
if artifact_clean:
|
||||
self.artifact_clean()
|
||||
if repo_clean:
|
||||
self.repo_clean()
|
||||
|
||||
def status(self):
|
||||
return all((item.status() for item in self.items))
|
|
@ -1,48 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Http(Object):
|
||||
__typename__ = 'http'
|
||||
|
||||
def __init__(self, env, name, http_root, port, network,
|
||||
status_url='/status', shutdown_url='/shutdown'):
|
||||
self.name = name
|
||||
self.env = env
|
||||
self.http_root = http_root
|
||||
self.port = port
|
||||
self.network = network
|
||||
self.status_url = status_url
|
||||
self.shutdown_url = shutdown_url
|
||||
|
||||
def start(self):
|
||||
if not self.status():
|
||||
LOG.debug('Starting HTTP server')
|
||||
self.env.driver.http_start(self)
|
||||
|
||||
def stop(self):
|
||||
if self.status():
|
||||
LOG.debug('Stopping HTTP server')
|
||||
self.env.driver.http_stop(self)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.http_status(self)
|
||||
LOG.debug('HTTP status %s' % status)
|
||||
return status
|
|
@ -1,45 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Net(Object):
|
||||
__typename__ = 'net'
|
||||
|
||||
def __init__(self, env, name, bridge, ip, forward):
|
||||
self.env = env
|
||||
self.name = name
|
||||
self.bridge = bridge
|
||||
self.ip = ip
|
||||
self.forward = forward
|
||||
|
||||
def start(self):
|
||||
if not self.status():
|
||||
LOG.debug('Starting network %s' % self.name)
|
||||
self.env.driver.net_start(self)
|
||||
|
||||
def stop(self):
|
||||
if self.status():
|
||||
LOG.debug('Stopping network %s' % self.name)
|
||||
self.env.driver.net_stop(self)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.net_status(self)
|
||||
LOG.debug('Network %s status %s' % (self.name, status))
|
||||
return status
|
|
@ -1,45 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Repo(Object):
|
||||
__typename__ = 'repo'
|
||||
|
||||
def __init__(self, env, name, url, path, branch='master'):
|
||||
self.env = env
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.path = path
|
||||
self.branch = branch
|
||||
|
||||
def clone(self):
|
||||
if not self.status():
|
||||
LOG.debug('Cloning repo %s' % self.name)
|
||||
self.env.driver.repo_clone(self)
|
||||
|
||||
def clean(self):
|
||||
if self.status():
|
||||
LOG.debug('Cleaning repo %s' % self.name)
|
||||
self.env.driver.repo_clean(self)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.repo_status(self)
|
||||
LOG.debug('Repo %s status %s' % (self.name, status))
|
||||
return status
|
|
@ -1,79 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Ssh(Object):
|
||||
__typename__ = 'ssh'
|
||||
|
||||
def __init__(self, env, name, host, key_filename, user='root',
|
||||
connection_timeout=5, command_timeout=10):
|
||||
self.env = env
|
||||
self.name = name
|
||||
self.host = host
|
||||
self.user = user
|
||||
self.key_filename = key_filename
|
||||
self.connection_timeout = int(connection_timeout)
|
||||
self.command_timeout = int(command_timeout)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.ssh_status(self)
|
||||
LOG.debug('SSH %s status %s' % (self.name, status))
|
||||
return status
|
||||
|
||||
def put_content(self, content, remote_filename):
|
||||
if self.status():
|
||||
LOG.debug('Putting content %s' % self.name)
|
||||
self.env.driver.ssh_put_content(self, content, remote_filename)
|
||||
else:
|
||||
raise Exception('Wrong ssh status: %s' % self.name)
|
||||
|
||||
def get_file(self, remote_filename, filename):
|
||||
if self.status():
|
||||
LOG.debug('Getting file %s' % self.name)
|
||||
self.env.driver.ssh_get_file(self, remote_filename, filename)
|
||||
else:
|
||||
raise Exception('Wrong ssh status: %s' % self.name)
|
||||
|
||||
def put_file(self, filename, remote_filename):
|
||||
if self.status():
|
||||
LOG.debug('Putting file %s' % self.name)
|
||||
self.env.driver.ssh_put_file(self, filename, remote_filename)
|
||||
else:
|
||||
raise Exception('Wrong ssh status: %s' % self.name)
|
||||
|
||||
def run(self, command, command_timeout=None):
|
||||
if self.status():
|
||||
LOG.debug('Running command %s' % self.name)
|
||||
return self.env.driver.ssh_run(
|
||||
self, command, command_timeout or self.command_timeout)
|
||||
raise Exception('Wrong ssh status: %s' % self.name)
|
||||
|
||||
def wait(self, timeout=200):
|
||||
begin_time = time.time()
|
||||
# this loop does not have sleep statement
|
||||
# because it relies on self.connection_timeout
|
||||
# which is by default 5 seconds
|
||||
while time.time() - begin_time < timeout:
|
||||
if self.status():
|
||||
return True
|
||||
LOG.debug('Waiting for ssh connection to be '
|
||||
'available: %s' % self.name)
|
||||
return False
|
|
@ -1,44 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Tftp(Object):
|
||||
__typename__ = 'tftp'
|
||||
|
||||
def __init__(self, env, name, tftp_root, network):
|
||||
self.name = name
|
||||
self.env = env
|
||||
self.tftp_root = tftp_root
|
||||
self.network = network
|
||||
|
||||
def start(self):
|
||||
if not self.status():
|
||||
LOG.debug('Starting TFTP')
|
||||
self.env.driver.tftp_start(self)
|
||||
|
||||
def stop(self):
|
||||
if self.status():
|
||||
LOG.debug('Stopping TFTP')
|
||||
self.env.driver.tftp_stop(self)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.tftp_status(self)
|
||||
LOG.debug('TFTP status %s' % status)
|
||||
return status
|
|
@ -1,74 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from fuel_agent_ci.objects import Object
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Vm(Object):
|
||||
__typename__ = 'vm'
|
||||
|
||||
def __init__(self, env, name, boot=None):
|
||||
self.env = env
|
||||
self.name = name
|
||||
self.interfaces = []
|
||||
self.disks = []
|
||||
self.boot = boot or 'hd'
|
||||
|
||||
def add_interface(self, **kwargs):
|
||||
if 'interface' in kwargs:
|
||||
interface = kwargs['interface']
|
||||
else:
|
||||
interface = Interface(**kwargs)
|
||||
self.interfaces.append(interface)
|
||||
return interface
|
||||
|
||||
def add_disk(self, **kwargs):
|
||||
if 'disk' in kwargs:
|
||||
disk = kwargs['disk']
|
||||
else:
|
||||
disk = Disk(**kwargs)
|
||||
self.disks.append(disk)
|
||||
return disk
|
||||
|
||||
def start(self):
|
||||
if not self.status():
|
||||
LOG.debug('Starting virtual machine %s' % self.name)
|
||||
self.env.driver.vm_start(self)
|
||||
|
||||
def stop(self):
|
||||
if self.status():
|
||||
LOG.debug('Stopping virtual machine %s' % self.name)
|
||||
self.env.driver.vm_stop(self)
|
||||
|
||||
def status(self):
|
||||
status = self.env.driver.vm_status(self)
|
||||
LOG.debug('Virtual machine %s status %s' % (self.name, status))
|
||||
return status
|
||||
|
||||
|
||||
class Interface(object):
|
||||
def __init__(self, mac, network, model_type='e1000'):
|
||||
self.mac = mac
|
||||
self.network = network
|
||||
self.model_type = model_type
|
||||
|
||||
|
||||
class Disk(object):
|
||||
def __init__(self, size=None, base=None):
|
||||
self.size = size
|
||||
self.base = base
|
|
@ -1,13 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
|
@ -1,105 +0,0 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import jinja2
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from unittest.case import TestCase
|
||||
except ImportError:
|
||||
# Runing unit-tests in production environment
|
||||
from unittest2.case import TestCase
|
||||
import yaml
|
||||
|
||||
from fuel_agent_ci.objects import environment
|
||||
from fuel_agent_ci import utils
|
||||
|
||||
# FIXME(kozhukalov) it is better to set this as command line arg
|
||||
ENV_FILE = os.path.join(os.path.dirname(__file__),
|
||||
'../../samples/ci_environment.yaml')
|
||||
|
||||
|
||||
class BaseFuelAgentCITest(TestCase):
|
||||
FUEL_AGENT_REPO_NAME = 'fuel_agent'
|
||||
FUEL_AGENT_HTTP_NAME = 'http'
|
||||
FUEL_AGENT_NET_NAME = 'net'
|
||||
FUEL_AGENT_DHCP_NAME = 'dhcp'
|
||||
FUEL_AGENT_SSH_NAME = 'vm'
|
||||
FUEL_AGENT_TEMPLATE_PATH = '/usr/share/fuel-agent/cloud-init-templates'
|
||||
|
||||
def setUp(self):
|
||||
super(BaseFuelAgentCITest, self).setUp()
|
||||
|
||||
# Starting environment
|
||||
with open(ENV_FILE) as f:
|
||||
ENV_DATA = (yaml.load(f.read()))
|
||||
self.env = environment.Environment.new(**ENV_DATA)
|
||||
self.env.start()
|
||||
|
||||
self.repo = self.env.repo_by_name(self.FUEL_AGENT_REPO_NAME)
|
||||
self.ssh = self.env.ssh_by_name(self.FUEL_AGENT_SSH_NAME)
|
||||
self.http = self.env.http_by_name(self.FUEL_AGENT_HTTP_NAME)
|
||||
self.dhcp_hosts = self.env.dhcp_by_name(self.FUEL_AGENT_DHCP_NAME).hosts
|
||||
self.net = self.env.net_by_name(self.FUEL_AGENT_NET_NAME)
|
||||
|
||||
self.ssh.wait()
|
||||
self._upgrade_fuel_agent()
|
||||
|
||||
def _upgrade_fuel_agent(self):
|
||||
"""This method is to be deprecated when artifact
|
||||
based build system is ready.
|
||||
"""
|
||||
src_dir = os.path.join(self.env.envdir, self.repo.path, 'fuel_agent')
|
||||
package_name = 'fuel-agent-0.1.0.tar.gz'
|
||||
|
||||
# Building fuel-agent pip package
|
||||
utils.execute('python setup.py sdist', cwd=src_dir)
|
||||
|
||||
# Putting fuel-agent pip package on a node
|
||||
self.ssh.put_file(
|
||||
os.path.join(src_dir, 'dist', package_name),
|
||||
os.path.join('/tmp', package_name))
|
||||
|
||||
# Installing fuel_agent pip package
|
||||
self.ssh.run('pip install --upgrade %s' %
|
||||
os.path.join('/tmp', package_name))
|
||||
|
||||
# Copying fuel_agent templates
|
||||
self.ssh.run('mkdir -p %s' % self.FUEL_AGENT_TEMPLATE_PATH)
|
||||
for f in os.listdir(
|
||||
os.path.join(src_dir, 'cloud-init-templates')):
|
||||
if f.endswith('.jinja2'):
|
||||
self.ssh.put_file(
|
||||
os.path.join(src_dir, 'cloud-init-templates', f),
|
||||
os.path.join(self.FUEL_AGENT_TEMPLATE_PATH, f))
|
||||
|
||||
self.ssh.put_file(
|
||||
os.path.join(src_dir, 'etc/fuel-agent/fuel-agent.conf.sample'),
|
||||
'/etc/fuel-agent/fuel-agent.conf')
|
||||
|
||||
def tearDown(self):
|
||||
super(BaseFuelAgentCITest, self).tearDown()
|
||||
self.env.stop()
|
||||
|
||||
def render_template(self,
|
||||
template_name,
|
||||
template_dir=os.path.join(os.path.dirname(__file__),
|
||||
'templates'),
|
||||
template_data=None):
|
||||
env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
|
||||
template = env.get_template(template_name)
|
||||
return template.render(**(template_data or {}))
|
|
@ -1,100 +0,0 @@
|
|||
#cloud-boothook
|
||||
#!/bin/bash
|
||||
|
||||
function add_str_to_file_if_not_exists {
|
||||
file=$1
|
||||
str=$2
|
||||
val=$3
|
||||
if ! grep -q "^ *${str}" $file; then
|
||||
echo $val >> $file
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
cloud-init-per instance disable_selinux_on_the_fly setenforce 0
|
||||
|
||||
cloud-init-per instance disable_selinux sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/sysconfig/selinux
|
||||
|
||||
|
||||
# configure udev rules
|
||||
|
||||
# udev persistent net
|
||||
cloud-init-per instance udev_persistent_net1 service network stop
|
||||
|
||||
DEFAULT_GW={{ MASTER_IP }}
|
||||
ADMIN_MAC={{ ADMIN_MAC }}
|
||||
ADMIN_IF=$(echo {{ UDEVRULES }} | sed 's/[,=]/\n/g' | grep "$ADMIN_MAC" | cut -d_ -f2 | head -1)
|
||||
INSTALL_IF=$(ifconfig | grep "$ADMIN_MAC" | head -1 | cut -d' ' -f1)
|
||||
NETADDR=( $(ifconfig $INSTALL_IF | grep -oP "[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}") )
|
||||
if [ ! -z "$(grep $ADMIN_IF /etc/sysconfig/network-scripts/ifcfg-$ADMIN_IF | grep dhcp)" ] ; then
|
||||
echo -e "# FROM COBBLER SNIPPET\nDEVICE=$ADMIN_IF\nIPADDR=${NETADDR[0]}\nNETMASK=${NETADDR[2]}\nBOOTPROTO=none\nONBOOT=yes\nUSERCTL=no\n" > /etc/sysconfig/network-scripts/ifcfg-"$ADMIN_IF"
|
||||
fi
|
||||
|
||||
cloud-init-per instance set_gateway echo GATEWAY="$DEFAULT_GW" | tee -a /etc/sysconfig/network
|
||||
|
||||
#Add static udev rules
|
||||
cloud-init-per instance udev_persistent_net2 echo {{ UDEVRULES }} | tr ' ' '\n' | grep udevrules | tr '[:upper:]' '[:lower:]' | sed -e 's/udevrules=//g' -e 's/,/\n/g' | sed -e "s/^/SUBSYSTEM==\"net\",\ ACTION==\"add\",\ DRIVERS==\"?*\",\ ATTR{address}==\"/g" -e "s/_/\",\ ATTR{type}==\"1\",\ KERNEL==\"eth*\",\ NAME=\"/g" -e "s/$/\"/g" | tee /etc/udev/rules.d/70-persistent-net.rules
|
||||
|
||||
cloud-init-per instance udev_persistent_net3 udevadm control --reload-rules
|
||||
cloud-init-per instance udev_persistent_net4 udevadm trigger --attr-match=subsystem=net
|
||||
cloud-init-per instance udev_persistent_net5 service network start
|
||||
|
||||
# end of udev
|
||||
|
||||
# configure black module lists
|
||||
# virt-what should be installed
|
||||
if [ ! -f /etc/modprobe.d/blacklist-i2c_piix4.conf ]; then
|
||||
([[ $(virt-what) = "virtualbox" ]] && echo "blacklist i2c_piix4" >> /etc/modprobe.d/blacklist-i2c_piix4.conf || :)
|
||||
modprobe -r i2c_piix4
|
||||
fi
|
||||
|
||||
cloud-init-per instance conntrack_ipv4 echo nf_conntrack_ipv4 | tee -a /etc/rc.modules
|
||||
cloud-init-per instance conntrack_ipv6 echo nf_conntrack_ipv6 | tee -a /etc/rc.modules
|
||||
cloud-init-per instance chmod_rc_modules chmod +x /etc/rc.modules
|
||||
cloud-init-per instance conntrack_max echo "net.nf_conntrack_max=1048576" | tee -a /etc/sysctl.conf
|
||||
|
||||
cloud-init-per instance conntrack_ipv4_load modprobe nf_conntrack_ipv4
|
||||
cloud-init-per instance conntrack_ipv6_load modprobe nf_conntrack_ipv6
|
||||
cloud-init-per instance conntrack_max_set sysctl -w "net.nf_conntrack_max=1048576"
|
||||
|
||||
cloud-init-per instance mkdir_coredump mkdir -p /var/log/coredump
|
||||
cloud-init-per instance set_coredump echo -e "kernel.core_pattern=/var/log/coredump/core.%e.%p.%h.%t" | tee -a /etc/sysctl.conf
|
||||
cloud-init-per instance set_chmod chmod 777 /var/log/coredump
|
||||
cloud-init-per instance set_limits echo -e "* soft core unlimited\n* hard core unlimited" | tee -a /etc/security/limits.conf
|
||||
|
||||
|
||||
#NOTE: disabled for centos?
|
||||
#cloud-init-per instance dhclient echo 'supersede routers 0;' | tee /etc/dhcp/dhclient.conf
|
||||
|
||||
# ntp sync
|
||||
cloud-init-per instance service ntp stop | tee /dev/null
|
||||
cloud-init-per instance sync_date ntpdate -t 4 -b {{ MASTER_IP }}
|
||||
cloud-init-per instance sync_hwclock hwclock --systohc
|
||||
|
||||
cloud-init-per instance edit_ntp_conf1 sed -i '/^\s*tinker panic/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf2 sed -i '1 i tinker panic 0' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf3 echo 0 > /var/lib/ntp/drift
|
||||
cloud-init-per instance edit_ntp_conf_0 chown ntp: /var/lib/ntp/drift
|
||||
cloud-init-per instance edit_ntp_conf3 sed -i '/^\s*server/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf4 echo "server {{ MASTER_IP }} burst iburst" | tee -a /etc/ntp.conf
|
||||
|
||||
|
||||
# Point installed ntpd to Master node
|
||||
cloud-init-per instance set_ntpdate sed -i 's/SYNC_HWCLOCK\s*=\s*no/SYNC_HWCLOCK=yes/' /etc/sysconfig/ntpdate
|
||||
cloud-init-per instance set_ntpd_0 chkconfig ntpd on
|
||||
cloud-init-per instance set_ntpd_1 chkconfig ntpdate on
|
||||
|
||||
cloud-init-per instance removeUseDNS sed -i --follow-symlinks -e '/UseDNS/d' /etc/ssh/sshd_config
|
||||
add_str_to_file_if_not_exists /etc/ssh/sshd_config 'UseDNS' 'UseDNS no'
|
||||
|
||||
cloud-init-per instance gssapi_disable sed -i -e "/^\s*GSSAPICleanupCredentials yes/d" -e "/^\s*GSSAPIAuthentication yes/d" /etc/ssh/sshd_config
|
||||
|
||||
cloud-init-per instance nailgun_agent echo 'flock -w 0 -o /var/lock/agent.lock -c "/opt/nailgun/bin/agent >> /var/log/nailgun-agent.log 2>&1"' | tee /etc/rc.local
|
||||
|
||||
# Copying default bash settings to the root directory
|
||||
cloud-init-per instance skel_bash cp -f /etc/skel/.bash* /root/
|
||||
|
||||
cloud-init-per instance hiera_puppet mkdir -p /etc/puppet /var/lib/hiera
|
||||
cloud-init-per instance touch_puppet touch /var/lib/hiera/common.yaml /etc/puppet/hiera.yaml
|
||||
|
||||
cloud-init-per instance clean_repos find /etc/yum.repos.d/. -name '*.repo' -delete
|
|
@ -1,74 +0,0 @@
|
|||
#cloud-boothook
|
||||
#!/bin/bash
|
||||
|
||||
function add_str_to_file_if_not_exists {
|
||||
file=$1
|
||||
str=$2
|
||||
val=$3
|
||||
if ! grep -q "^ *${str}" $file; then
|
||||
echo $val >> $file
|
||||
fi
|
||||
}
|
||||
|
||||
# configure udev rules
|
||||
|
||||
# udev persistent net
|
||||
cloud-init-per instance udev_persistent_net1 /etc/init.d/networking stop
|
||||
|
||||
ADMIN_MAC={{ ADMIN_MAC }}
|
||||
ADMIN_IF=$(echo {{ UDEVRULES }} | sed 's/[,=]/\n/g' | grep "$ADMIN_MAC" | cut -d_ -f2 | head -1)
|
||||
INSTALL_IF=$(ifconfig | grep "$ADMIN_MAC" | head -1 | cut -d' ' -f1)
|
||||
# Check if we do not already have static config (or interface seems unconfigured)
|
||||
NETADDR=( $(ifconfig $INSTALL_IF | grep -oP "[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}") )
|
||||
if [ ! -z "$(grep $ADMIN_IF /etc/network/interfaces.d/ifcfg-$ADMIN_IF | grep dhcp)" ] ; then
|
||||
echo -e "auto $ADMIN_IF\niface $ADMIN_IF inet static\n\taddress ${NETADDR[0]}\n\tnetmask ${NETADDR[2]}\n\tbroadcast ${NETADDR[1]}" > /etc/network/interfaces.d/ifcfg-"$ADMIN_IF"
|
||||
fi
|
||||
#Add static udev rules
|
||||
cloud-init-per instance udev_persistent_net2 echo {{ UDEVRULES }} | tr ' ' '\n' | grep udevrules | tr '[:upper:]' '[:lower:]' | sed -e 's/udevrules=//g' -e 's/,/\n/g' | sed -e "s/^/SUBSYSTEM==\"net\",\ ACTION==\"add\",\ DRIVERS==\"?*\",\ ATTR{address}==\"/g" -e "s/_/\",\ ATTR{type}==\"1\",\ KERNEL==\"eth*\",\ NAME=\"/g" -e "s/$/\"/g" | tee /etc/udev/rules.d/70-persistent-net.rules
|
||||
|
||||
cloud-init-per instance udev_persistent_net3 udevadm control --reload-rules
|
||||
cloud-init-per instance udev_persistent_net4 udevadm trigger --attr-match=subsystem=net
|
||||
cloud-init-per instance udev_persistent_net5 /etc/init.d/networking start
|
||||
|
||||
# end of udev
|
||||
|
||||
# configure black module lists
|
||||
# virt-what should be installed
|
||||
if [ ! -f /etc/modprobe.d/blacklist-i2c_piix4.conf ]; then
|
||||
([[ $(virt-what) = "virtualbox" ]] && echo "blacklist i2c_piix4" >> /etc/modprobe.d/blacklist-i2c_piix4.conf || :) && update-initramfs -u -k all
|
||||
modprobe -r i2c_piix4
|
||||
fi
|
||||
|
||||
cloud-init-per instance conntrack_ipv4 echo nf_conntrack_ipv4 | tee -a /etc/modules
|
||||
cloud-init-per instance conntrack_ipv6 echo nf_conntrack_ipv6 | tee -a /etc/modules
|
||||
cloud-init-per instance conntrack_max echo "net.nf_conntrack_max=1048576" | tee -a /etc/sysctl.conf
|
||||
|
||||
cloud-init-per instance conntrack_ipv4_load modprobe nf_conntrack_ipv4
|
||||
cloud-init-per instance conntrack_ipv6_load modprobe nf_conntrack_ipv6
|
||||
cloud-init-per instance conntrack_max_set sysctl -w "net.nf_conntrack_max=1048576"
|
||||
|
||||
cloud-init-per instance dhclient echo 'supersede routers 0;' | tee /etc/dhcp/dhclient.conf
|
||||
|
||||
# ntp sync
|
||||
cloud-init-per instance service ntp stop | tee /dev/null
|
||||
cloud-init-per instance sync_date ntpdate -t 4 -b {{ MASTER_IP }}
|
||||
cloud-init-per instance sync_hwclock hwclock --systohc
|
||||
|
||||
cloud-init-per instance edit_ntp_conf1 sed -i '/^\s*tinker panic/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf2 sed -i '1 i tinker panic 0' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf3 echo 0 > /var/lib/ntp/drift
|
||||
cloud-init-per instance edit_ntp_conf3 sed -i '/^\s*server/ d' /etc/ntp.conf
|
||||
cloud-init-per instance edit_ntp_conf4 echo "server {{ MASTER_IP }} burst iburst" | tee -a /etc/ntp.conf
|
||||
|
||||
cloud-init-per instance removeUseDNS sed -i --follow-symlinks -e '/UseDNS/d' /etc/ssh/sshd_config
|
||||
add_str_to_file_if_not_exists /etc/ssh/sshd_config 'UseDNS' 'UseDNS no'
|
||||
|
||||
cloud-init-per instance gssapi_disable sed -i -e "/^\s*GSSAPICleanupCredentials yes/d" -e "/^\s*GSSAPIAuthentication yes/d" /etc/ssh/sshd_config
|
||||
|
||||
cloud-init-per instance nailgun_agent echo 'flock -w 0 -o /var/lock/agent.lock -c "/opt/nailgun/bin/agent >> /var/log/nailgun-agent.log 2>&1"' | tee /etc/rc.local
|
||||
|
||||
# Copying default bash settings to the root directory
|
||||
cloud-init-per instance skel_bash cp -f /etc/skel/.bash* /root/
|
||||
|
||||
cloud-init-per instance hiera_puppet mkdir -p /etc/puppet /var/lib/hiera
|
||||
cloud-init-per instance touch_puppet touch /var/lib/hiera/common.yaml /etc/puppet/hiera.yaml
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue