devstack: add pci-sim fake SR-IOV PCI passthrough test fixture
Testing Cyborg's pci_driver and Nova PCI passthrough normally requires physical SR-IOV hardware. This change introduces pci-sim, an out-of-tree Linux kernel module (fake_pci_sriov) that creates fake SR-IOV PCI devices whose virtual functions can be bound to VFIO and assigned to QEMU guests. This makes it possible to exercise the full Nova and Cyborg PCI passthrough control plane, including device enumeration, VF assignment, and VFIO passthrough to CirrOS guests, in a standard devstack VM with no physical SR-IOV hardware present. pci-sim/ Source for the fake_pci_sriov out-of-tree kernel module. The module creates one fake PCI host bridge per PF, each with software-emulated VFs backed by a 16550-style UART loopback payload. Build artifacts are excluded via .gitignore. The directory is excluded from Python wheels and sdists via [tool.setuptools.packages.find]. devstack/ lib/pci_sim provides functions to build, load, configure, and unload the module. All paths are expressed relative to CYBORG_DIR so they resolve correctly within the cyborg checkout. devstack/settings gains a new ENABLE_PCI_SIM master toggle (default False) and the full set of PCI_SIM_* variables to control the fake PCI topology and service integration. PCI_SIM_CONFIGURE_NOVA_PCI and PCI_SIM_CONFIGURE_CYBORG_PCI are both defaulted to ENABLE_PCI_SIM but provide a explicy opt out if required. plugin.sh drives the pci-sim lifecycle alongside cyborg's own phases. The pci-sim cyborg service configuration is applied before start_cyborg in the extra phase so the agent picks up pci_driver and passthrough_whitelist on first start. devstack/local-conf.pci-sim.sample provides a ready-to-use local.conf for a two-PF, four-VF-per-PF topology with both Nova and Cyborg integrations enabled. tools/ check-kernel-config.sh validates the running kernel's config against the requirements for fake_pci_sriov before attempting a build. run-devstack-serial-echo-test.sh creates test VMs using the pci-sim flavors and validates the UART echo path inside a CirrOS guest. doc/source/contributor/pci-sim/ New contributor documentation covering build instructions, devstack setup, plugin settings, testing procedures, and kernel config requirements. Linked from the contributor index. Assisted-By: pi gpt-5.5 Assisted-By: pi sonnet-4.6 Change-Id: I14dba5d0de833c9f2df28478daacbf8250de2127 Signed-off-by: Sean Mooney <work@seanmooney.info>
This commit is contained in:
Sean Mooney
+12
@@ -22,6 +22,18 @@ etc/cyborg/cyborg.conf.sample
|
||||
.idea/*
|
||||
.vscode/*
|
||||
|
||||
# pci-sim kernel module build outputs
|
||||
pci-sim/*.o
|
||||
pci-sim/*.ko
|
||||
pci-sim/*.mod
|
||||
pci-sim/*.mod.c
|
||||
pci-sim/*.cmd
|
||||
pci-sim/modules.order
|
||||
pci-sim/Module.symvers
|
||||
pci-sim/.module-common.o
|
||||
pci-sim/.tmp_versions/
|
||||
pci-sim/.cache.mk
|
||||
|
||||
# Editors
|
||||
*.sublime-workspace
|
||||
*.swp
|
||||
|
||||
+25
-28
@@ -1,15 +1,17 @@
|
||||
- job:
|
||||
name: cyborg-tempest-stub
|
||||
name: cyborg-tempest-base
|
||||
abstract: true
|
||||
description: |
|
||||
Abstract base job for Cyborg CI. Defines shared devstack
|
||||
configuration, irrelevant file patterns, and required projects
|
||||
used by both tempest and grenade jobs.
|
||||
base job of all cyborg tempest jobs
|
||||
parent: devstack-tempest
|
||||
timeout: 7800
|
||||
nodeset: devstack-single-node-debian-trixie
|
||||
required-projects: &base_required_projects
|
||||
- openstack/cyborg
|
||||
- openstack/python-cyborgclient
|
||||
- openstack/cyborg-tempest-plugin
|
||||
vars: &base_vars
|
||||
- openstack/tempest
|
||||
vars: &cyborg_base_vars
|
||||
devstack_plugins: &cyborg_devstack_plugins
|
||||
cyborg: https://opendev.org/openstack/cyborg
|
||||
devstack_services: &cyborg_devstack_services
|
||||
@@ -23,6 +25,8 @@
|
||||
c-sch: false
|
||||
c-vol: false
|
||||
cinder: false
|
||||
devstack_localrc:
|
||||
ENABLE_PCI_SIM: true
|
||||
tempest_plugins:
|
||||
- cyborg-tempest-plugin
|
||||
tempest_test_regex: cyborg_tempest_plugin
|
||||
@@ -44,7 +48,10 @@
|
||||
current release(N).
|
||||
parent: grenade
|
||||
required-projects: *base_required_projects
|
||||
vars: *base_vars
|
||||
vars: &cyborg_grenade_vars
|
||||
<<: *cyborg_base_vars
|
||||
devstack_localrc:
|
||||
ENABLE_PCI_SIM: false
|
||||
irrelevant-files: *base_irrelevant_files
|
||||
|
||||
- job:
|
||||
@@ -56,26 +63,16 @@
|
||||
would be 2026.1.
|
||||
parent: grenade-skip-level-always
|
||||
required-projects: *base_required_projects
|
||||
vars: *base_vars
|
||||
vars: *cyborg_grenade_vars
|
||||
irrelevant-files: *base_irrelevant_files
|
||||
|
||||
- job:
|
||||
name: cyborg-multinode-tempest
|
||||
parent: devstack-tempest
|
||||
nodeset: openstack-two-node-noble
|
||||
abstract: true
|
||||
parent: cyborg-tempest-base
|
||||
nodeset: devstack-two-node-debian-trixie
|
||||
description: |
|
||||
Cyborg multinode devstack tempest job
|
||||
required-projects: *base_required_projects
|
||||
irrelevant-files: *base_irrelevant_files
|
||||
vars:
|
||||
devstack_plugins: *cyborg_devstack_plugins
|
||||
devstack_services:
|
||||
<<: *cyborg_devstack_services
|
||||
cyborg-agent: false
|
||||
tempest_plugins:
|
||||
- cyborg-tempest-plugin
|
||||
tempest_test_regex: cyborg_tempest_plugin
|
||||
tox_envlist: all
|
||||
Abstract multinode Cyborg devstack tempest job
|
||||
group-vars:
|
||||
subnode:
|
||||
devstack_plugins: *cyborg_devstack_plugins
|
||||
@@ -84,27 +81,26 @@
|
||||
cyborg-api: false
|
||||
cyborg-cond: false
|
||||
tempest: false
|
||||
devstack_localrc:
|
||||
PCI_SIM_CREATE_TEST_FLAVORS: false
|
||||
ENABLE_PCI_SIM: true
|
||||
|
||||
- job:
|
||||
name: cyborg-tempest-py3
|
||||
description: |
|
||||
Cyborg devstack tempest tests job for >=Ussuri release
|
||||
run on python 3.
|
||||
parent: devstack-tempest
|
||||
timeout: 7800
|
||||
required-projects: *base_required_projects
|
||||
vars: *base_vars
|
||||
irrelevant-files: *base_irrelevant_files
|
||||
|
||||
parent: cyborg-multinode-tempest
|
||||
- job:
|
||||
name: cyborg-tempest-ipv6
|
||||
parent: devstack-tempest-ipv6
|
||||
nodeset: devstack-single-node-debian-trixie
|
||||
description: |
|
||||
Cyborg devstack tempest tests job for IPv6-only deployment
|
||||
voting: true
|
||||
timeout: 7800
|
||||
required-projects: *base_required_projects
|
||||
vars: *base_vars
|
||||
vars: *cyborg_base_vars
|
||||
irrelevant-files: *base_irrelevant_files
|
||||
|
||||
- project:
|
||||
@@ -123,5 +119,6 @@
|
||||
gate:
|
||||
jobs:
|
||||
- cyborg-tempest-py3
|
||||
- cyborg-tempest-ipv6
|
||||
- cyborg-grenade
|
||||
- cyborg-grenade-skip-level-always
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
prune pci-sim
|
||||
prune devstack
|
||||
+3
-3
@@ -137,7 +137,7 @@ function configure_cyborg_agent {
|
||||
iniset $CYBORG_CONF_FILE agent enabled_drivers $CYBORG_ENABLED_DRIVERS
|
||||
|
||||
# Configure placement client auth for multinode deployments
|
||||
configure_keystoneauth $CYBORG_CONF_FILE placement placement
|
||||
configure_keystoneauth $CYBORG_CONF_FILE cyborg placement
|
||||
|
||||
# Add retries so that placement registers the resource provider
|
||||
# Set higher than the default (3) because in CI the subnode agent
|
||||
@@ -183,8 +183,8 @@ function configure_cyborg_conductor {
|
||||
|
||||
# this one is needed for lookup of Cyborg API endpoint via Keystone
|
||||
configure_auth_for service_catalog
|
||||
configure_keystoneauth $CYBORG_CONF_FILE nova nova
|
||||
configure_keystoneauth $CYBORG_CONF_FILE placement placement
|
||||
configure_keystoneauth $CYBORG_CONF_FILE cyborg nova
|
||||
configure_keystoneauth $CYBORG_CONF_FILE cyborg placement
|
||||
|
||||
# NOTE(sean-k-mooney) it is not obvious why this is in the conductor
|
||||
# section or what sets is_deployed_by_agent so this should either be
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# pci-sim exposes a deliberately small fake SR-IOV topology: each fake
|
||||
# PF can create up to seven VFs because the module models one PCI slot
|
||||
# with function 0 as the PF and functions 1-7 as VFs. Keep the IDs here
|
||||
# in sync with pci-sim/fake_pci_sriov.h and the Cyborg PCI driver tests.
|
||||
PCI_SIM_VENDOR_ID=${PCI_SIM_VENDOR_ID:-0x1d55}
|
||||
PCI_SIM_PF_DEVICE_ID=${PCI_SIM_PF_DEVICE_ID:-0x1000}
|
||||
PCI_SIM_VF_DEVICE_ID=${PCI_SIM_VF_DEVICE_ID:-0x1001}
|
||||
PCI_SIM_MODULE_NAME=${PCI_SIM_MODULE_NAME:-fake_pci_sriov}
|
||||
|
||||
function pci_sim_validate_config {
|
||||
# The module supports up to 16 fake host bridges/PFs and up to 7 VFs
|
||||
# per PF. Validate early so stack.sh fails before loading the module
|
||||
# with a topology the driver will reject.
|
||||
if [[ $PCI_SIM_NUM_PFS -lt 1 || $PCI_SIM_NUM_PFS -gt 16 ]]; then
|
||||
die $LINENO "PCI_SIM_NUM_PFS must be between 1 and 16; got $PCI_SIM_NUM_PFS"
|
||||
fi
|
||||
|
||||
if [[ $PCI_SIM_NUM_VFS -lt 0 || $PCI_SIM_NUM_VFS -gt 7 ]]; then
|
||||
die $LINENO "PCI_SIM_NUM_VFS must be between 0 and 7; got $PCI_SIM_NUM_VFS"
|
||||
fi
|
||||
}
|
||||
|
||||
function pci_sim_sysfs_write {
|
||||
local value=$1
|
||||
local path=$2
|
||||
|
||||
# Most lifecycle operations are sysfs writes that require root. Keep the
|
||||
# sudo/tee pattern in one helper so callers describe intent instead of
|
||||
# repeating privilege plumbing.
|
||||
echo "$value" | sudo tee "$path" >/dev/null
|
||||
}
|
||||
|
||||
function pci_sim_find_devs {
|
||||
local device_id=$1
|
||||
local dev
|
||||
|
||||
# Discover devices through sysfs rather than lspci output so the function
|
||||
# works before pciutils is installed and returns stable BDF names for later
|
||||
# sysfs writes.
|
||||
for dev in /sys/bus/pci/devices/*; do
|
||||
[[ -e $dev/vendor && -e $dev/device ]] || continue
|
||||
[[ $(cat "$dev/vendor") == "$PCI_SIM_VENDOR_ID" ]] || continue
|
||||
[[ $(cat "$dev/device") == "$device_id" ]] || continue
|
||||
basename "$dev"
|
||||
done | sort
|
||||
}
|
||||
|
||||
function pci_sim_csv_has_value {
|
||||
local csv=$1
|
||||
local value=$2
|
||||
|
||||
# Strip spaces and pad with leading/trailing commas so a single pattern
|
||||
# matches whole CSV tokens at the beginning, middle, or end. This avoids
|
||||
# partial matches such as "nova" matching "nova_flavor".
|
||||
[[ ,${csv// /}, == *,${value},* ]]
|
||||
}
|
||||
|
||||
function pci_sim_iniappend_csv_unique {
|
||||
local file=$1
|
||||
local section=$2
|
||||
local option=$3
|
||||
local value=$4
|
||||
local current
|
||||
|
||||
current=$(iniget "$file" "$section" "$option" || true)
|
||||
# DevStack functions may call this more than once across rechecks or
|
||||
# restacks. Preserve existing values and append only missing filters.
|
||||
if [[ -z $current ]]; then
|
||||
iniset "$file" "$section" "$option" "$value"
|
||||
elif ! pci_sim_csv_has_value "$current" "$value"; then
|
||||
iniset "$file" "$section" "$option" "$current,$value"
|
||||
fi
|
||||
}
|
||||
|
||||
function pci_sim_openstack_auth {
|
||||
# Resource creation happens late enough that openstackclient is available,
|
||||
# but standalone phase testing may not have sourced credentials yet.
|
||||
if [[ -n ${OS_AUTH_URL:-} ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -n ${TOP_DIR:-} && -f $TOP_DIR/openrc ]]; then
|
||||
source $TOP_DIR/openrc admin admin
|
||||
elif [[ -n ${DEST:-} && -f $DEST/devstack/openrc ]]; then
|
||||
source $DEST/devstack/openrc admin admin
|
||||
fi
|
||||
}
|
||||
|
||||
function pci_sim_find_vfs_for_pf {
|
||||
local pf=$1
|
||||
local dev
|
||||
|
||||
# The PCI core exposes a VF's parent PF through the physfn symlink. Use it
|
||||
# to keep Nova/Cyborg assignment stable when more than one fake PF exists.
|
||||
for dev in /sys/bus/pci/devices/*; do
|
||||
[[ -e $dev/vendor && -e $dev/device && -e $dev/physfn ]] || continue
|
||||
[[ $(cat "$dev/vendor") == "$PCI_SIM_VENDOR_ID" ]] || continue
|
||||
[[ $(cat "$dev/device") == "$PCI_SIM_VF_DEVICE_ID" ]] || continue
|
||||
[[ $(basename "$(readlink -f "$dev/physfn")") == "$pf" ]] || continue
|
||||
basename "$dev"
|
||||
done | sort
|
||||
}
|
||||
|
||||
function pci_sim_bind_vf_to_vfio {
|
||||
local vf=$1
|
||||
local dev_path=/sys/bus/pci/devices/$vf
|
||||
|
||||
# driver_override forces this VF to the pci-sim-specific VFIO driver even
|
||||
# when the host loopback driver would otherwise match the same device ID.
|
||||
if [[ -e $dev_path/driver/unbind ]]; then
|
||||
pci_sim_sysfs_write "$vf" "$dev_path/driver/unbind"
|
||||
fi
|
||||
pci_sim_sysfs_write pci_sim_vfio_pci "$dev_path/driver_override"
|
||||
pci_sim_sysfs_write "$vf" /sys/bus/pci/drivers_probe
|
||||
|
||||
local driver
|
||||
driver=$(basename "$(readlink -f "$dev_path/driver")")
|
||||
if [[ $driver != pci_sim_vfio_pci ]]; then
|
||||
die $LINENO "Failed to bind $vf to pci_sim_vfio_pci; current driver is $driver"
|
||||
fi
|
||||
}
|
||||
|
||||
function build_pci_sim {
|
||||
set -x
|
||||
local kver=$(uname -r)
|
||||
local kconfig=/boot/config-${kver}
|
||||
local -a pkgs=(build-essential linux-headers-${kver} kmod)
|
||||
|
||||
# CONFIG_DEBUG_INFO_BTF causes the kernel Makefile to invoke pahole
|
||||
# (from dwarves) to generate BTF type info for the .ko. Read the
|
||||
# running kernel's own config so this adapts automatically to any
|
||||
# kernel on any distro without needing changes here.
|
||||
if grep -q '^CONFIG_DEBUG_INFO_BTF=y' "${kconfig}"; then
|
||||
pkgs+=(dwarves)
|
||||
fi
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y "${pkgs[@]}"
|
||||
|
||||
$CYBORG_DIR/tools/check-kernel-config.sh
|
||||
make -C $CYBORG_DIR/pci-sim modules
|
||||
sudo make -C $CYBORG_DIR/pci-sim modules_install
|
||||
sudo depmod -a
|
||||
}
|
||||
|
||||
function pci_sim_configure_vfio_iommu {
|
||||
set -x
|
||||
|
||||
[[ "$PCI_SIM_ALLOW_UNSAFE_INTERRUPTS" == True ]] || return 0
|
||||
|
||||
# Nested/devstack hosts often lack interrupt remapping. In that case VFIO
|
||||
# refuses to attach groups unless vfio_iommu_type1 allows unsafe interrupts.
|
||||
# Pass the module parameter before loading VFIO and also update the runtime
|
||||
# sysfs knob for repeated stack.sh runs where the module is already loaded.
|
||||
sudo modprobe vfio_iommu_type1 allow_unsafe_interrupts=1 || true
|
||||
if [[ -e /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts ]]; then
|
||||
pci_sim_sysfs_write Y /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts
|
||||
fi
|
||||
}
|
||||
|
||||
function load_pci_sim {
|
||||
set -x
|
||||
pci_sim_validate_config
|
||||
|
||||
# Start from a deterministic state so repeated stack.sh runs do not inherit
|
||||
# stale VFs or module parameters.
|
||||
unload_pci_sim || true
|
||||
|
||||
pci_sim_configure_vfio_iommu
|
||||
sudo modprobe vfio-pci
|
||||
# Prefer the installed module path used by DevStack. Fall back to the
|
||||
# build-tree module to make local plugin-function testing convenient.
|
||||
# shellcheck disable=SC2086
|
||||
if ! sudo modprobe $PCI_SIM_MODULE_NAME num_pfs=$PCI_SIM_NUM_PFS $PCI_SIM_MODULE_ARGS; then
|
||||
# shellcheck disable=SC2086
|
||||
sudo insmod $CYBORG_DIR/pci-sim/fake_pci_sriov.ko \
|
||||
num_pfs=$PCI_SIM_NUM_PFS $PCI_SIM_MODULE_ARGS
|
||||
fi
|
||||
}
|
||||
|
||||
function configure_pci_sim_vfs {
|
||||
set -x
|
||||
local pf vf
|
||||
local -a pfs vfs
|
||||
|
||||
# Reset sriov_numvfs before recreating VFs so repeated stack.sh runs do not
|
||||
# leave stale VF devices or driver bindings behind.
|
||||
pci_sim_validate_config
|
||||
mapfile -t pfs < <(pci_sim_find_devs "$PCI_SIM_PF_DEVICE_ID")
|
||||
|
||||
if [[ ${#pfs[@]} -ne $PCI_SIM_NUM_PFS ]]; then
|
||||
die $LINENO "Expected $PCI_SIM_NUM_PFS pci-sim PF(s), found ${#pfs[@]}: ${pfs[*]}"
|
||||
fi
|
||||
|
||||
for pf in "${pfs[@]}"; do
|
||||
pci_sim_sysfs_write 0 "/sys/bus/pci/devices/$pf/sriov_numvfs" || true
|
||||
pci_sim_sysfs_write "$PCI_SIM_NUM_VFS" "/sys/bus/pci/devices/$pf/sriov_numvfs"
|
||||
mapfile -t vfs < <(pci_sim_find_vfs_for_pf "$pf")
|
||||
for vf in "${vfs[@]}"; do
|
||||
pci_sim_bind_vf_to_vfio "$vf"
|
||||
done
|
||||
done
|
||||
|
||||
echo "pci-sim PFs: ${pfs[*]}"
|
||||
echo "pci-sim VFs: $(pci_sim_find_devs "$PCI_SIM_VF_DEVICE_ID" | xargs echo)"
|
||||
}
|
||||
|
||||
function pci_sim_pf_target {
|
||||
local index=$1
|
||||
|
||||
# When both services are enabled, split fake PFs by index. This lets one
|
||||
# DevStack cloud expose Nova PCI passthrough and Cyborg accelerator flows
|
||||
# at the same time without both services managing the same VF.
|
||||
if [[ "$PCI_SIM_CONFIGURE_NOVA_PCI" == True && \
|
||||
"$PCI_SIM_CONFIGURE_CYBORG_PCI" == True ]]; then
|
||||
if (( index % 2 == 0 )); then
|
||||
echo nova
|
||||
else
|
||||
echo cyborg
|
||||
fi
|
||||
elif [[ "$PCI_SIM_CONFIGURE_NOVA_PCI" == True ]]; then
|
||||
echo nova
|
||||
elif [[ "$PCI_SIM_CONFIGURE_CYBORG_PCI" == True ]]; then
|
||||
echo cyborg
|
||||
fi
|
||||
}
|
||||
|
||||
function configure_pci_sim_nova_conf_file {
|
||||
local conf_file=$1
|
||||
shift
|
||||
local -a specs aliases
|
||||
local vf
|
||||
|
||||
[[ -f $conf_file ]] || die $LINENO "Nova config does not exist: $conf_file"
|
||||
|
||||
for vf in "$@"; do
|
||||
specs+=("{\"address\":\"$vf\",\"vendor_id\":\"1d55\",\"product_id\":\"1001\",\"device_type\":\"type-VF\",\"managed\":\"no\"}")
|
||||
done
|
||||
aliases+=("{\"vendor_id\":\"1d55\",\"product_id\":\"1001\",\"device_type\":\"type-VF\",\"name\":\"$PCI_SIM_NOVA_ALIAS_NAME\"}")
|
||||
|
||||
if (( ${#specs[@]} > 0 )); then
|
||||
iniset_multiline "$conf_file" pci device_spec "${specs[@]}"
|
||||
iniset_multiline "$conf_file" pci alias "${aliases[@]}"
|
||||
|
||||
iniset "$conf_file" filter_scheduler available_filters \
|
||||
nova.scheduler.filters.all_filters
|
||||
pci_sim_iniappend_csv_unique "$conf_file" filter_scheduler \
|
||||
enabled_filters PciPassthroughFilter
|
||||
pci_sim_iniappend_csv_unique "$conf_file" filter_scheduler \
|
||||
enabled_filters NUMATopologyFilter
|
||||
|
||||
if [[ "$PCI_SIM_NOVA_PCI_IN_PLACEMENT" == True ]]; then
|
||||
iniset "$conf_file" pci report_in_placement True
|
||||
iniset "$conf_file" filter_scheduler pci_in_placement True
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function configure_pci_sim_nova {
|
||||
set -x
|
||||
local conf_file
|
||||
local -a nova_conf_files
|
||||
|
||||
[[ "$PCI_SIM_CONFIGURE_NOVA_PCI" == True ]] || return 0
|
||||
|
||||
NOVA_CONF=${NOVA_CONF:-/etc/nova/nova.conf}
|
||||
NOVA_CPU_CONF=${NOVA_CPU_CONF:-/etc/nova/nova-cpu.conf}
|
||||
|
||||
# Some DevStack deployments keep compute-service PCI options in a separate
|
||||
# nova-cpu.conf. Update both files when they exist so nova-api and
|
||||
# nova-compute see compatible alias/device_spec settings.
|
||||
nova_conf_files=("$NOVA_CONF")
|
||||
if [[ -f $NOVA_CPU_CONF && $NOVA_CPU_CONF != "$NOVA_CONF" ]]; then
|
||||
nova_conf_files+=("$NOVA_CPU_CONF")
|
||||
fi
|
||||
|
||||
for conf_file in "${nova_conf_files[@]}"; do
|
||||
configure_pci_sim_nova_conf_file "$conf_file" "$@"
|
||||
done
|
||||
}
|
||||
|
||||
function configure_pci_sim_cyborg {
|
||||
set -x
|
||||
local current_drivers
|
||||
local -a specs
|
||||
local vf
|
||||
|
||||
[[ "$PCI_SIM_CONFIGURE_CYBORG_PCI" == True ]] || return 0
|
||||
|
||||
CYBORG_CONF_FILE=${CYBORG_CONF_FILE:-/etc/cyborg/cyborg.conf}
|
||||
[[ -f $CYBORG_CONF_FILE ]] || die $LINENO "CYBORG_CONF_FILE does not exist: $CYBORG_CONF_FILE"
|
||||
|
||||
current_drivers=$(iniget "$CYBORG_CONF_FILE" agent enabled_drivers || true)
|
||||
# Enable the generic PCI driver without dropping any other accelerator
|
||||
# drivers the local DevStack environment already requested.
|
||||
if [[ -z $current_drivers ]]; then
|
||||
iniset "$CYBORG_CONF_FILE" agent enabled_drivers pci_driver
|
||||
elif [[ ,$current_drivers, != *,pci_driver,* ]]; then
|
||||
iniset "$CYBORG_CONF_FILE" agent enabled_drivers "$current_drivers,pci_driver"
|
||||
fi
|
||||
|
||||
for vf in "$@"; do
|
||||
specs+=("{\"address\":\"$vf\"}")
|
||||
done
|
||||
|
||||
if (( ${#specs[@]} > 0 )); then
|
||||
iniset_multiline "$CYBORG_CONF_FILE" pci passthrough_whitelist "${specs[@]}"
|
||||
fi
|
||||
}
|
||||
|
||||
function pci_sim_collect_target_vfs {
|
||||
local wanted_target=$1
|
||||
local pf target
|
||||
local -a pfs pf_vfs
|
||||
local index=0
|
||||
|
||||
mapfile -t pfs < <(pci_sim_find_devs "$PCI_SIM_PF_DEVICE_ID")
|
||||
for pf in "${pfs[@]}"; do
|
||||
target=$(pci_sim_pf_target "$index")
|
||||
if [[ $target == "$wanted_target" ]]; then
|
||||
mapfile -t pf_vfs < <(pci_sim_find_vfs_for_pf "$pf")
|
||||
printf '%s
|
||||
' "${pf_vfs[@]}"
|
||||
fi
|
||||
index=$((index + 1))
|
||||
done
|
||||
}
|
||||
|
||||
function configure_pci_sim_nova_service_config {
|
||||
set -x
|
||||
local -a nova_vfs
|
||||
|
||||
[[ "$PCI_SIM_CONFIGURE_NOVA_PCI" == True ]] || return 0
|
||||
mapfile -t nova_vfs < <(pci_sim_collect_target_vfs nova)
|
||||
configure_pci_sim_nova "${nova_vfs[@]}"
|
||||
echo "pci-sim Nova VFs: ${nova_vfs[*]:-}"
|
||||
}
|
||||
|
||||
function configure_pci_sim_cyborg_service_config {
|
||||
set -x
|
||||
local -a cyborg_vfs
|
||||
|
||||
[[ "$PCI_SIM_CONFIGURE_CYBORG_PCI" == True ]] || return 0
|
||||
mapfile -t cyborg_vfs < <(pci_sim_collect_target_vfs cyborg)
|
||||
configure_pci_sim_cyborg "${cyborg_vfs[@]}"
|
||||
echo "pci-sim Cyborg VFs: ${cyborg_vfs[*]:-}"
|
||||
}
|
||||
|
||||
function pci_sim_flavor_create_like_m1_nano {
|
||||
local flavor_name=$1
|
||||
local ram disk vcpus ephemeral swap
|
||||
|
||||
ram=$(openstack flavor show m1.nano -f value -c ram 2>/dev/null || echo 192)
|
||||
disk=$(openstack flavor show m1.nano -f value -c disk 2>/dev/null || echo 1)
|
||||
vcpus=$(openstack flavor show m1.nano -f value -c vcpus 2>/dev/null || echo 1)
|
||||
ephemeral=$(openstack flavor show m1.nano -f value \
|
||||
-c OS-FLV-EXT-DATA:ephemeral 2>/dev/null || echo 0)
|
||||
swap=$(openstack flavor show m1.nano -f value -c swap 2>/dev/null || echo 0)
|
||||
|
||||
if ! openstack flavor show "$flavor_name" >/dev/null 2>&1; then
|
||||
openstack flavor create "$flavor_name" --ram "$ram" \
|
||||
--disk "$disk" --vcpus "$vcpus" --ephemeral "$ephemeral" \
|
||||
--swap "$swap"
|
||||
fi
|
||||
openstack flavor set "$flavor_name" --property hw_rng:allowed=True
|
||||
}
|
||||
|
||||
function create_pci_sim_nova_flavor {
|
||||
[[ "$PCI_SIM_CONFIGURE_NOVA_PCI" == True ]] || return 0
|
||||
|
||||
pci_sim_flavor_create_like_m1_nano "$PCI_SIM_NOVA_FLAVOR_NAME"
|
||||
openstack flavor set "$PCI_SIM_NOVA_FLAVOR_NAME" \
|
||||
--property "pci_passthrough:alias=$PCI_SIM_NOVA_ALIAS_NAME:$PCI_SIM_NOVA_ALIAS_COUNT"
|
||||
}
|
||||
|
||||
function create_pci_sim_cyborg_device_profile {
|
||||
local groups
|
||||
|
||||
[[ "$PCI_SIM_CONFIGURE_CYBORG_PCI" == True ]] || return 0
|
||||
|
||||
groups='[{"resources:CUSTOM_PCI":"1","trait:CUSTOM_PCI_PRODUCT_ID_1001":"required"}]'
|
||||
if ! openstack accelerator device profile list -f value -c name | \
|
||||
grep -qx "$PCI_SIM_CYBORG_DEVICE_PROFILE_NAME"; then
|
||||
openstack accelerator device profile create \
|
||||
"$PCI_SIM_CYBORG_DEVICE_PROFILE_NAME" "$groups"
|
||||
fi
|
||||
}
|
||||
|
||||
function create_pci_sim_cyborg_flavor {
|
||||
[[ "$PCI_SIM_CONFIGURE_CYBORG_PCI" == True ]] || return 0
|
||||
|
||||
pci_sim_flavor_create_like_m1_nano "$PCI_SIM_CYBORG_FLAVOR_NAME"
|
||||
openstack flavor set "$PCI_SIM_CYBORG_FLAVOR_NAME" \
|
||||
--property "accel:device_profile=$PCI_SIM_CYBORG_DEVICE_PROFILE_NAME"
|
||||
}
|
||||
|
||||
function create_pci_sim_test_resources {
|
||||
set -x
|
||||
|
||||
# These resources are convenience fixtures for manual validation. The
|
||||
# helper is idempotent so unstack/stack cycles can reuse names safely.
|
||||
[[ "$PCI_SIM_CREATE_TEST_FLAVORS" == True ]] || return 0
|
||||
pci_sim_openstack_auth
|
||||
create_pci_sim_nova_flavor
|
||||
create_pci_sim_cyborg_device_profile
|
||||
create_pci_sim_cyborg_flavor
|
||||
}
|
||||
|
||||
function unload_pci_sim {
|
||||
set -x
|
||||
local cleanup=$CYBORG_DIR/pci-sim/cleanup_fake_pci_sriov.sh
|
||||
if [[ -x $cleanup ]]; then
|
||||
$cleanup || true
|
||||
elif grep -q "^$PCI_SIM_MODULE_NAME " /proc/modules; then
|
||||
sudo rmmod "$PCI_SIM_MODULE_NAME"
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
# Sample local.conf for Cyborg development with pci-sim fake SR-IOV devices.
|
||||
#
|
||||
# pci-sim builds and loads the fake_pci_sriov kernel module, which creates
|
||||
# fake SR-IOV PCI devices that can be passed through to QEMU guests via VFIO.
|
||||
# This allows Nova PCI passthrough and Cyborg accelerator testing without
|
||||
# physical SR-IOV hardware.
|
||||
#
|
||||
# Copy this file to the root of your DevStack checkout as local.conf and
|
||||
# adjust passwords and HOST_IP for your environment.
|
||||
#
|
||||
# pci-sim is enabled by default when the cyborg plugin is loaded.
|
||||
# Set ENABLE_PCI_SIM=False to disable it.
|
||||
#
|
||||
# NOTE: For nested/VM-based DevStack environments (the common case for
|
||||
# development), ensure the VM has nested KVM and IOMMU enabled, or set
|
||||
# PCI_SIM_ALLOW_UNSAFE_INTERRUPTS=True (the default) to allow VFIO without
|
||||
# interrupt remapping support.
|
||||
|
||||
[[local|localrc]]
|
||||
|
||||
ADMIN_PASSWORD=password
|
||||
DATABASE_PASSWORD=password
|
||||
RABBIT_PASSWORD=password
|
||||
SERVICE_PASSWORD=password
|
||||
|
||||
# Set this to the address other services/guests should use.
|
||||
# HOST_IP=192.0.2.10
|
||||
|
||||
# Enable Cyborg. pci-sim is included — no separate enable_plugin needed.
|
||||
enable_plugin cyborg https://opendev.org/openstack/cyborg
|
||||
|
||||
ENABLE_PCI_SIM=True
|
||||
# pci-sim topology: 2 fake PFs, 4 VFs each.
|
||||
# With both integrations enabled, PF0 VFs go to Nova and PF1 VFs to Cyborg.
|
||||
PCI_SIM_NUM_PFS=2
|
||||
PCI_SIM_NUM_VFS=4
|
||||
# optionally disable service integration
|
||||
# PCI_SIM_CONFIGURE_NOVA_PCI=False
|
||||
# PCI_SIM_CONFIGURE_CYBORG_PCI=False
|
||||
|
||||
# Required for nested/devstack environments without interrupt remapping.
|
||||
PCI_SIM_ALLOW_UNSAFE_INTERRUPTS=True
|
||||
|
||||
+33
-1
@@ -4,7 +4,9 @@
|
||||
# https://docs.openstack.org/devstack/latest/plugins.html
|
||||
|
||||
echo_summary "cyborg devstack plugin.sh called: $1/$2"
|
||||
source $DEST/cyborg/devstack/lib/cyborg
|
||||
CYBORG_DEVSTACK_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
source $CYBORG_DEVSTACK_DIR/lib/cyborg
|
||||
source $CYBORG_DEVSTACK_DIR/lib/pci_sim
|
||||
|
||||
case $1 in
|
||||
"stack")
|
||||
@@ -16,6 +18,12 @@ case $1 in
|
||||
echo_summary "Installing Cyborg"
|
||||
install_cyborg
|
||||
install_cyborg_client
|
||||
if [[ "$ENABLE_PCI_SIM" == True ]]; then
|
||||
install_package pciutils
|
||||
if [[ "$PCI_SIM_BUILD" == True ]]; then
|
||||
async_runfunc build_pci_sim
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
"post-config")
|
||||
# stack/post-config - Called after the layer 0 and 2 services
|
||||
@@ -24,10 +32,25 @@ case $1 in
|
||||
echo_summary "Configuring Cyborg"
|
||||
configure_cyborg
|
||||
create_cyborg_accounts
|
||||
if [[ "$ENABLE_PCI_SIM" == True ]]; then
|
||||
if [[ "$PCI_SIM_BUILD" == True ]]; then
|
||||
async_wait build_pci_sim
|
||||
fi
|
||||
if [[ "$PCI_SIM_LOAD" == True ]]; then
|
||||
load_pci_sim
|
||||
configure_pci_sim_vfs
|
||||
configure_pci_sim_nova_service_config
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
"extra")
|
||||
# stack/extra - Called near the end after layer 1 and 2
|
||||
# services have been started.
|
||||
# Configure pci-sim Cyborg service settings before starting
|
||||
# the agent so it picks up pci_driver and passthrough_whitelist.
|
||||
if [[ "$ENABLE_PCI_SIM" == True && "$PCI_SIM_LOAD" == True ]]; then
|
||||
configure_pci_sim_cyborg_service_config
|
||||
fi
|
||||
# Initialize cyborg
|
||||
init_cyborg
|
||||
# Start the cyborg API and cyborg taskmgr components
|
||||
@@ -41,16 +64,25 @@ case $1 in
|
||||
echo_summary "Configuring Tempest for Cyborg needs"
|
||||
cyborg_configure_tempest
|
||||
fi
|
||||
if [[ "$ENABLE_PCI_SIM" == True ]]; then
|
||||
create_pci_sim_test_resources
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
"unstack")
|
||||
# unstack - Called by unstack.sh before other services are shut down.
|
||||
stop_cyborg
|
||||
if [[ "$ENABLE_PCI_SIM" == True ]]; then
|
||||
unload_pci_sim || true
|
||||
fi
|
||||
;;
|
||||
"clean")
|
||||
# clean - Called by clean.sh before other services are cleaned, but after
|
||||
# unstack.sh has been called.
|
||||
cleanup_cyborg
|
||||
if [[ "$ENABLE_PCI_SIM" == True ]]; then
|
||||
unload_pci_sim || true
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -43,3 +43,54 @@ CYBORG_CLIENT_INSTALL=$(trueorfalse True CYBORG_CLIENT_INSTALL)
|
||||
CYBORG_CLIENT_REPO=${CYBORG_CLIENT_REPO:-"$GIT_BASE/openstack/python-cyborgclient"}
|
||||
CYBORG_CLIENT_BRANCH=${CYBORG_CLIENT_BRANCH:-master}
|
||||
CYBORG_CLIENT_DIR="${DEST}/python-cyborgclient"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# pci-sim: fake SR-IOV kernel module for PCI passthrough testing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Master switch. Set to False to skip all pci-sim phases (build, load,
|
||||
# configuration, and test-resource creation). Defaults to False for backward
|
||||
# compatibility with old jobs.
|
||||
ENABLE_PCI_SIM=$(trueorfalse False ENABLE_PCI_SIM)
|
||||
|
||||
# Whether to build and install pci-sim during DevStack.
|
||||
PCI_SIM_BUILD=$(trueorfalse ${ENABLE_PCI_SIM} PCI_SIM_BUILD)
|
||||
|
||||
# Whether to load fake_pci_sriov after building it.
|
||||
PCI_SIM_LOAD=$(trueorfalse ${ENABLE_PCI_SIM} PCI_SIM_LOAD)
|
||||
|
||||
# Whether to allow VFIO on hosts without interrupt remapping support. This is
|
||||
# required for nested/devstack test environments where vfio_iommu_type1 would
|
||||
# otherwise reject attaching pci-sim VFs with "No interrupt remapping support".
|
||||
PCI_SIM_ALLOW_UNSAFE_INTERRUPTS=$(trueorfalse ${ENABLE_PCI_SIM} PCI_SIM_ALLOW_UNSAFE_INTERRUPTS)
|
||||
|
||||
# Number of fake SR-IOV PFs to create when PCI_SIM_LOAD=True.
|
||||
PCI_SIM_NUM_PFS=${PCI_SIM_NUM_PFS:-2}
|
||||
|
||||
# Number of VFs to enable on each fake PF when PCI_SIM_LOAD=True.
|
||||
# fake_pci_sriov currently supports 0 through 7 VFs per PF.
|
||||
PCI_SIM_NUM_VFS=${PCI_SIM_NUM_VFS:-4}
|
||||
|
||||
# Whether this plugin should configure Nova to consume pci-sim VFs directly via
|
||||
# [pci] device_spec/alias.
|
||||
PCI_SIM_CONFIGURE_NOVA_PCI=$(trueorfalse ${ENABLE_PCI_SIM} PCI_SIM_CONFIGURE_NOVA_PCI)
|
||||
|
||||
# Whether this plugin should configure Cyborg's generic pci_driver to manage
|
||||
# pci-sim VFs via [agent] enabled_drivers and [pci] passthrough_whitelist.
|
||||
PCI_SIM_CONFIGURE_CYBORG_PCI=$(trueorfalse ${ENABLE_PCI_SIM} PCI_SIM_CONFIGURE_CYBORG_PCI)
|
||||
|
||||
# Whether to configure Nova to report PCI inventory to Placement and to use
|
||||
# Placement for PCI scheduling. Only used when PCI_SIM_CONFIGURE_NOVA_PCI=True.
|
||||
PCI_SIM_NOVA_PCI_IN_PLACEMENT=$(trueorfalse ${ENABLE_PCI_SIM} PCI_SIM_NOVA_PCI_IN_PLACEMENT)
|
||||
|
||||
# Whether to create convenience test flavors/device profiles in stack/test-config.
|
||||
PCI_SIM_CREATE_TEST_FLAVORS=$(trueorfalse ${ENABLE_PCI_SIM} PCI_SIM_CREATE_TEST_FLAVORS)
|
||||
PCI_SIM_NOVA_FLAVOR_NAME=${PCI_SIM_NOVA_FLAVOR_NAME:-pci-sim-nova}
|
||||
PCI_SIM_NOVA_ALIAS_NAME=${PCI_SIM_NOVA_ALIAS_NAME:-pci_sim_vf}
|
||||
PCI_SIM_NOVA_ALIAS_COUNT=${PCI_SIM_NOVA_ALIAS_COUNT:-1}
|
||||
PCI_SIM_CYBORG_DEVICE_PROFILE_NAME=${PCI_SIM_CYBORG_DEVICE_PROFILE_NAME:-pci-sim-cyborg-dp}
|
||||
PCI_SIM_CYBORG_FLAVOR_NAME=${PCI_SIM_CYBORG_FLAVOR_NAME:-pci-sim-cyborg}
|
||||
|
||||
# Extra module parameters used when PCI_SIM_LOAD=True. Do not include num_pfs;
|
||||
# set PCI_SIM_NUM_PFS instead.
|
||||
PCI_SIM_MODULE_ARGS=${PCI_SIM_MODULE_ARGS:-}
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
aaS
|
||||
filp
|
||||
|
||||
@@ -46,6 +46,9 @@ Reviewing
|
||||
|
||||
* :doc:`/contributor/release-guide`: Chronological guide for release liaisons
|
||||
|
||||
* :doc:`/contributor/pci-sim/index`: pci-sim fake SR-IOV kernel module for
|
||||
PCI passthrough testing without physical hardware
|
||||
|
||||
.. # NOTE: toctree needs to be placed at the end of the section to
|
||||
# keep the document structure in the PDF doc.
|
||||
.. toctree::
|
||||
@@ -60,3 +63,4 @@ Reviewing
|
||||
grenade-upgrade
|
||||
driver-development-guide
|
||||
release-guide
|
||||
pci-sim/index
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
=====
|
||||
Build
|
||||
=====
|
||||
|
||||
Local build
|
||||
===========
|
||||
|
||||
Install headers for the running kernel, then build the module from the
|
||||
repository root:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ sudo apt install linux-headers-$(uname -r) build-essential
|
||||
$ bash tools/check-kernel-config.sh
|
||||
$ make -C pci-sim modules
|
||||
|
||||
The module is created at ``pci-sim/fake_pci_sriov.ko``.
|
||||
|
||||
The build does not require a Linux git checkout. It uses the kernel build
|
||||
tree provided by the running kernel's header package:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ make -C /lib/modules/$(uname -r)/build M=$PWD/pci-sim modules
|
||||
@@ -0,0 +1,75 @@
|
||||
========
|
||||
DevStack
|
||||
========
|
||||
|
||||
pci-sim is built into the Cyborg DevStack plugin. No separate
|
||||
``enable_plugin`` line is needed — enabling Cyborg is sufficient:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
enable_plugin cyborg https://opendev.org/openstack/cyborg
|
||||
|
||||
PCI_SIM_CONFIGURE_NOVA_PCI=True
|
||||
PCI_SIM_CONFIGURE_CYBORG_PCI=True
|
||||
|
||||
A ready-to-use sample configuration is provided in the Cyborg repository at
|
||||
``devstack/local-conf.pci-sim.sample``.
|
||||
|
||||
Plugin settings
|
||||
===============
|
||||
|
||||
``ENABLE_PCI_SIM``
|
||||
Master switch. Set to ``False`` to skip all pci-sim phases (build, load,
|
||||
configuration, and test-resource creation). Defaults to ``False``.
|
||||
|
||||
``PCI_SIM_BUILD``
|
||||
Build and install the module during the DevStack ``install`` phase.
|
||||
Defaults to ``True``.
|
||||
|
||||
``PCI_SIM_LOAD``
|
||||
Load ``fake_pci_sriov`` after building it. Defaults to ``True``.
|
||||
|
||||
``PCI_SIM_ALLOW_UNSAFE_INTERRUPTS``
|
||||
Allow VFIO on hosts without interrupt remapping support by setting
|
||||
``vfio_iommu_type1.allow_unsafe_interrupts`` before probing the fake VFs.
|
||||
Defaults to ``True`` for nested/devstack test environments.
|
||||
|
||||
``PCI_SIM_NUM_PFS``
|
||||
Number of fake SR-IOV physical functions to create. Defaults to ``2``.
|
||||
|
||||
``PCI_SIM_NUM_VFS``
|
||||
Number of virtual functions to enable on each fake PF. Defaults to ``4``.
|
||||
The current module supports ``0`` through ``7`` VFs per PF.
|
||||
|
||||
``PCI_SIM_CONFIGURE_NOVA_PCI``
|
||||
Configure Nova to consume pci-sim VFs directly using Nova's ``[pci]``
|
||||
``device_spec`` and ``alias`` options. Defaults to ``False``. Nova entries
|
||||
are generated by exact VF address and use ``managed=\"no\"``. The plugin
|
||||
updates both ``/etc/nova/nova.conf`` and, when present, DevStack's compute
|
||||
service config ``/etc/nova/nova-cpu.conf``.
|
||||
|
||||
``PCI_SIM_CONFIGURE_CYBORG_PCI``
|
||||
Configure Cyborg's generic ``pci_driver`` to manage pci-sim VFs using
|
||||
Cyborg's ``[agent] enabled_drivers`` and ``[pci] passthrough_whitelist``
|
||||
options. Defaults to ``False``. If both Nova and Cyborg configuration are
|
||||
enabled, the plugin assigns whole PFs in round-robin order: all VFs from PF0
|
||||
to Nova, all VFs from PF1 to Cyborg, and so on.
|
||||
|
||||
``PCI_SIM_NOVA_PCI_IN_PLACEMENT``
|
||||
When Nova PCI configuration is enabled, configure
|
||||
``[pci] report_in_placement`` and
|
||||
``[filter_scheduler] pci_in_placement``. Defaults to ``True``.
|
||||
|
||||
``PCI_SIM_CREATE_TEST_FLAVORS``
|
||||
Create convenience test resources during ``stack/test-config``. Defaults to
|
||||
``True``. When Nova PCI configuration is enabled, the plugin creates a
|
||||
flavor named by ``PCI_SIM_NOVA_FLAVOR_NAME`` with
|
||||
``pci_passthrough:alias``. When
|
||||
Cyborg PCI configuration is enabled, the plugin creates a Cyborg device
|
||||
profile named by ``PCI_SIM_CYBORG_DEVICE_PROFILE_NAME`` and a flavor named by
|
||||
``PCI_SIM_CYBORG_FLAVOR_NAME`` with ``accel:device_profile``.
|
||||
|
||||
``PCI_SIM_MODULE_ARGS``
|
||||
Optional extra arguments passed to ``modprobe fake_pci_sriov`` when
|
||||
``PCI_SIM_LOAD=True``. Do not include ``num_pfs`` here; use
|
||||
``PCI_SIM_NUM_PFS`` instead.
|
||||
@@ -0,0 +1,21 @@
|
||||
.. _pci-sim:
|
||||
|
||||
=======
|
||||
pci-sim
|
||||
=======
|
||||
|
||||
``pci-sim`` provides an out-of-tree build of the ``fake_pci_sriov`` Linux
|
||||
kernel module for DevStack and CI testing of SR-IOV PCI passthrough flows
|
||||
without physical SR-IOV hardware. It is included in the Cyborg repository
|
||||
under the top-level ``pci-sim/`` directory and is built and loaded
|
||||
automatically by the Cyborg DevStack plugin.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
overview
|
||||
build
|
||||
devstack
|
||||
testing
|
||||
kernel-dependencies
|
||||
migration-plan
|
||||
@@ -0,0 +1,36 @@
|
||||
===================
|
||||
Kernel dependencies
|
||||
===================
|
||||
|
||||
The original in-tree Kconfig dependency was::
|
||||
|
||||
PCI && PCI_DOMAINS && IOMMU_API && VFIO_PCI_CORE && TTY
|
||||
|
||||
As an out-of-tree module, these are checked against
|
||||
``/boot/config-$(uname -r)`` by ``tools/check-kernel-config.sh``.
|
||||
|
||||
Required options
|
||||
================
|
||||
|
||||
The following options must be enabled in the running kernel configuration:
|
||||
|
||||
* ``CONFIG_MODULES=y``
|
||||
* ``CONFIG_PCI=y``
|
||||
* ``CONFIG_PCI_DOMAINS=y``
|
||||
* ``CONFIG_IOMMU_API=y``
|
||||
* ``CONFIG_TTY=y``
|
||||
* ``CONFIG_VFIO=y`` or ``m``
|
||||
* ``CONFIG_VFIO_PCI_CORE=y`` or ``m``
|
||||
|
||||
Runtime/test options
|
||||
====================
|
||||
|
||||
The smoke tests are more useful when these are also enabled:
|
||||
|
||||
* ``CONFIG_PCI_IOV``
|
||||
* ``CONFIG_VFIO_PCI``
|
||||
* ``CONFIG_KVM``
|
||||
|
||||
The initial compatibility target is the current Ubuntu 26.04 generic kernel,
|
||||
``7.0.0-15-generic``. Broader Ubuntu 26.04 kernel update compatibility should
|
||||
be added by version-guarding specific API differences as they are found.
|
||||
@@ -0,0 +1,19 @@
|
||||
=====================
|
||||
Future migration work
|
||||
=====================
|
||||
|
||||
The fake VF device state is small: UART registers plus FIFO contents. A future
|
||||
migration implementation should preserve enough state that a guest's
|
||||
assigned VF continues to function after live migration.
|
||||
|
||||
Initial phases
|
||||
==============
|
||||
|
||||
#. Add UART state save/restore helpers independent of VFIO migration plumbing.
|
||||
#. Wire the current VFIO PCI migration APIs into ``pci_sim_vfio_pci``.
|
||||
#. Validate with matching fake PF/VF topology on source and destination hosts.
|
||||
#. Add Nova/libvirt level tests once QEMU behavior is understood.
|
||||
|
||||
This is intentionally future work. The first goal for this repository is a
|
||||
reliable out-of-tree build and local load/test flow for the current
|
||||
Ubuntu 26.04 kernel.
|
||||
@@ -0,0 +1,17 @@
|
||||
========
|
||||
Overview
|
||||
========
|
||||
|
||||
``fake_pci_sriov.ko`` is a self-contained test module for SR-IOV
|
||||
control-plane flows without physical SR-IOV hardware. It creates one or more
|
||||
fake PCI host bridges, each with one physical function and software-created
|
||||
virtual functions. The VFs can be bound to VFIO and assigned to a QEMU guest,
|
||||
which makes the module useful for OpenStack Nova, libvirt and QEMU testing.
|
||||
|
||||
The emulated VF payload is intentionally small: a 16550-style UART loopback.
|
||||
That is enough to prove that a VF assigned through VFIO is visible and usable
|
||||
inside an unmodified guest such as CirrOS.
|
||||
|
||||
The source currently lives under the top-level ``pci-sim/`` directory and is
|
||||
built as an out-of-tree kernel module against
|
||||
``/lib/modules/$(uname -r)/build``.
|
||||
@@ -0,0 +1,48 @@
|
||||
=======
|
||||
Testing
|
||||
=======
|
||||
|
||||
The local test environment assumes passwordless ``sudo``, matching typical
|
||||
DevStack usage.
|
||||
|
||||
Host loopback smoke test
|
||||
========================
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ make -C pci-sim modules
|
||||
$ sudo modprobe vfio-pci
|
||||
$ sudo insmod pci-sim/fake_pci_sriov.ko
|
||||
$ sudo pci-sim/test_pci_sim_loopback.py
|
||||
$ sudo pci-sim/cleanup_fake_pci_sriov.sh
|
||||
|
||||
QEMU/VFIO smoke test
|
||||
====================
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ make -C pci-sim modules
|
||||
$ MODULE=./pci-sim/fake_pci_sriov.ko pci-sim/run_fake_pci_qemu_vfio_smoke.sh
|
||||
|
||||
The QEMU helpers bind a fake VF to ``pci_sim_vfio_pci`` and verify that QEMU
|
||||
can start with the assigned VF. CirrOS-based helpers are also included for
|
||||
guest-level UART loopback testing.
|
||||
|
||||
DevStack Nova/Cyborg serial echo test
|
||||
======================================
|
||||
|
||||
After stacking with the pci-sim DevStack plugin and its generated test
|
||||
flavors, run the end-to-end helper from the repository root:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ tools/run-devstack-serial-echo-test.sh
|
||||
|
||||
The helper uses ``openstack --os-cloud devstack-admin`` by default. It
|
||||
creates one VM at a time on the ``private`` network using the
|
||||
``pci-sim-nova`` and ``pci-sim-cyborg`` flavors, opens the current
|
||||
project's default security group for ICMP and SSH, attaches a temporary
|
||||
floating IP, SSHes into CirrOS with the default ``cirros``/``gocubsgo``
|
||||
credentials, verifies the passed-through serial PCI device, checks the UART
|
||||
echo path, and then deletes the test VM and tagged floating IP. Use
|
||||
``--cleanup`` to remove leftovers from interrupted runs.
|
||||
@@ -0,0 +1,23 @@
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
|
||||
config SAMPLE_FAKE_PCI_SRIOV
|
||||
tristate "Fake PCI Host Controller with SR-IOV for testing"
|
||||
depends on PCI && PCI_DOMAINS && IOMMU_API && VFIO_PCI_CORE && TTY
|
||||
help
|
||||
This builds a sample kernel module that creates a fake PCI host
|
||||
controller with SR-IOV capable devices. It includes a software
|
||||
IOMMU driver that provides proper IOMMU groups, enabling the
|
||||
devices to be assigned to VMs via VFIO.
|
||||
|
||||
The module creates a PCI device that appears in lspci and can
|
||||
have up to 7 Virtual Functions enabled via sriov_numvfs sysfs.
|
||||
|
||||
This is useful for testing:
|
||||
- PCI device passthrough with libvirt/QEMU
|
||||
- SR-IOV functionality in OpenStack Nova
|
||||
- VFIO driver development
|
||||
|
||||
To compile this as a module, choose M here. The module will
|
||||
be called fake_pci_sriov.
|
||||
|
||||
If unsure, say N.
|
||||
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
|
||||
obj-m += fake_pci_sriov.o
|
||||
fake_pci_sriov-y := \
|
||||
fake_pci_sriov_core.o \
|
||||
fake_pci_sriov_cfg.o \
|
||||
fake_pci_sriov_iommu.o \
|
||||
fake_pci_sriov_uart.o \
|
||||
fake_pci_sriov_vfio.o
|
||||
|
||||
KDIR ?= /lib/modules/$(shell uname -r)/build
|
||||
PWD := $(CURDIR)
|
||||
|
||||
.PHONY: all modules clean install modules_install check-kernel-config \
|
||||
test test-loopback test-multi-pf
|
||||
|
||||
all: modules
|
||||
|
||||
modules:
|
||||
$(MAKE) -C $(KDIR) M=$(PWD) modules
|
||||
|
||||
clean:
|
||||
$(MAKE) -C $(KDIR) M=$(PWD) clean
|
||||
|
||||
install modules_install:
|
||||
$(MAKE) -C $(KDIR) M=$(PWD) modules_install
|
||||
depmod -a
|
||||
|
||||
check-kernel-config:
|
||||
../tools/check-kernel-config.sh
|
||||
|
||||
# Run host-side tests. Requires sudo and a kernel that supports module loading.
|
||||
# Build the module first with 'make modules'.
|
||||
test: test-loopback test-multi-pf
|
||||
|
||||
test-loopback:
|
||||
sudo -n python3 $(PWD)/test_pci_sim_loopback.py --reload --unload \
|
||||
--module $(PWD)/fake_pci_sriov.ko
|
||||
|
||||
test-multi-pf:
|
||||
bash $(PWD)/run_fake_pci_multi_pf_smoke.sh
|
||||
@@ -0,0 +1,59 @@
|
||||
Fake PCI SR-IOV VFIO test fixture
|
||||
=================================
|
||||
|
||||
``fake_pci_sriov.ko`` is a self-contained module for testing SR-IOV
|
||||
control-plane flows without physical SR-IOV hardware. It creates fake PCI host
|
||||
bridges with physical functions and software-created virtual functions.
|
||||
The VFs can be bound to VFIO and assigned to QEMU guests.
|
||||
|
||||
For the standalone repository build from the repository root with::
|
||||
|
||||
make
|
||||
|
||||
or from this directory with::
|
||||
|
||||
make -C /lib/modules/$(uname -r)/build M=$PWD modules
|
||||
|
||||
The built module is ``fake_pci_sriov.ko``.
|
||||
|
||||
Local helpers
|
||||
-------------
|
||||
|
||||
The helper scripts are intentionally kept next to the out-of-tree module so
|
||||
relative paths work in standalone development checkouts and DevStack jobs.
|
||||
|
||||
``cleanup_fake_pci_sriov.sh``
|
||||
Best-effort cleanup for fake VFs, fake PFs, driver overrides, and the
|
||||
loaded module. DevStack and smoke tests use this to make repeated runs
|
||||
deterministic.
|
||||
|
||||
``test_pci_sim_loopback.py``
|
||||
Host-side loopback test. It loads the module, enables VFs, waits for
|
||||
``/dev/ttyPCI_SIM<N>``, and checks that bytes written to each TTY are read
|
||||
back from the same TTY.
|
||||
|
||||
``run_fake_pci_multi_pf_smoke.sh``
|
||||
Host-side multi-PF smoke test. It verifies multiple fake PFs, expected
|
||||
class codes, VF creation, and cleanup.
|
||||
|
||||
``run_fake_pci_qemu_vfio_smoke.sh``
|
||||
Bounded QEMU/VFIO smoke test. It verifies that a fake VF can bind to
|
||||
``pci_sim_vfio_pci`` and that QEMU can start with the VF assigned.
|
||||
|
||||
``run_cirros_vfio_guest_probe.sh``
|
||||
CirrOS guest probe test. It logs in over the serial console and runs
|
||||
guest-side PCI and TTY discovery commands.
|
||||
|
||||
``run_cirros_vfio_userdata_echo.sh``
|
||||
CirrOS config-drive end-to-end test. Guest user-data probes the VF and
|
||||
verifies the emulated UART loopback path inside the guest.
|
||||
|
||||
``cirros_vfio_userdata_echo.sh``
|
||||
Guest-side user-data script copied into the config-drive generated by
|
||||
``run_cirros_vfio_userdata_echo.sh``.
|
||||
|
||||
Common test targets are available from this directory after building the
|
||||
module::
|
||||
|
||||
make test-loopback
|
||||
make test-multi-pf
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/bin/sh
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
#
|
||||
# CirrOS config-drive user-data for run_cirros_vfio_userdata_echo.sh.
|
||||
#
|
||||
# This script runs inside the CirrOS guest. It discovers the fake PCI VF that
|
||||
# QEMU passed through with VFIO, prints PCI and serial-driver diagnostics to the
|
||||
# serial console, and verifies that the emulated UART loopback path can echo an
|
||||
# alphabet string through one of the guest ttyS devices.
|
||||
|
||||
exec </dev/console >/dev/ttyS0 2>&1
|
||||
echo E2E_BEGIN
|
||||
PATH=/sbin:/bin:/usr/sbin:/usr/bin
|
||||
DEV=""
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
v=$(cat $d/vendor 2>/dev/null || true)
|
||||
p=$(cat $d/device 2>/dev/null || true)
|
||||
if { [ "$v" = "0x1d55" ] && [ "$p" = "0x1001" ]; } || { [ "$v" = "0x1d0f" ] && [ "$p" = "0x8250" ]; } || { [ "$v" = "0x10a9" ] && [ "$p" = "0x0003" ]; }; then DEV=$d; break; fi
|
||||
done
|
||||
echo DEV=$DEV
|
||||
if [ -n "$DEV" ]; then
|
||||
echo VENDOR=$(cat $DEV/vendor) DEVICE=$(cat $DEV/device) CLASS=$(cat $DEV/class)
|
||||
echo RESOURCE0=$(head -n1 $DEV/resource)
|
||||
BASE=$(awk 'NR==1 {print $1}' $DEV/resource)
|
||||
echo BASE=$BASE
|
||||
UART_OFFSET=0
|
||||
[ "$(cat $DEV/vendor)" = "0x10a9" ] && [ "$(cat $DEV/device)" = "0x0003" ] && UART_OFFSET=0x20178
|
||||
UART_BASE=$(printf "0x%x" $((BASE + UART_OFFSET)))
|
||||
echo UART_OFFSET=$UART_OFFSET UART_BASE=$UART_BASE
|
||||
else
|
||||
echo NO_FAKE_VF
|
||||
fi
|
||||
echo TTY_LIST_BEGIN
|
||||
ls -l /dev/ttyS* 2>&1 || true
|
||||
echo TTY_LIST_END
|
||||
echo DMESG_MATCH_BEGIN
|
||||
dmesg | grep -Ei '1d55|1001|1d0f|8250|10a9|0003|serial|ttyS' || true
|
||||
echo DMESG_MATCH_END
|
||||
if [ -n "$DEV" ] && command -v devmem >/dev/null 2>&1; then
|
||||
echo DEVMEM_PRESENT
|
||||
# 16550 offsets: THR/RBR=0, LSR=5. This is a smoke test for trapped MMIO.
|
||||
LSR_ADDR=$(printf "0x%x" $((UART_BASE + 5)))
|
||||
THR_ADDR=$(printf "0x%x" $((UART_BASE + 0)))
|
||||
echo LSR_BEFORE=$(devmem $LSR_ADDR 8 2>&1 || true)
|
||||
echo WRITE_THR_A=$(devmem $THR_ADDR 8 0x41 2>&1 || true)
|
||||
echo LSR_AFTER=$(devmem $LSR_ADDR 8 2>&1 || true)
|
||||
echo RBR_READ=$(devmem $THR_ADDR 8 2>&1 || true)
|
||||
echo TTY_ALPHABET_BEGIN
|
||||
TTY_PASS=0
|
||||
TEST_STR=ABCDEFGHIJKLMNOPQRSTUVWXYZ
|
||||
for t in /dev/ttyS32 /dev/ttyS33 /dev/ttyS34 /dev/ttyS35 /dev/ttyS1 /dev/ttyS2 /dev/ttyS3 /dev/ttyS4; do
|
||||
[ -e $t ] || continue
|
||||
echo TRY_TTY=$t
|
||||
if (
|
||||
exec 7<>$t || exit 1
|
||||
stty -F $t raw -echo -icanon clocal -hupcl min 0 time 20 9600 2>&1 || exit 1
|
||||
printf "%s" "$TEST_STR" >&7
|
||||
TTY_READ=$(dd bs=1 count=26 <&7 2>/dev/null || true)
|
||||
echo TTY_SELECTED=$t
|
||||
echo TTY_EXPECT=$TEST_STR
|
||||
echo TTY_READ=$TTY_READ
|
||||
echo TTY_READ_HEXDUMP_BEGIN
|
||||
printf "%s" "$TTY_READ" | hexdump -C || true
|
||||
echo TTY_READ_HEXDUMP_END
|
||||
[ "$TTY_READ" = "$TEST_STR" ] || exit 1
|
||||
exec 7>&-
|
||||
) 2>&1; then
|
||||
TTY_PASS=1
|
||||
echo TTY_ALPHABET_PASS=$t
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$TTY_PASS" != 1 ]; then
|
||||
echo TTY_ALPHABET_FAIL
|
||||
echo E2E_FAIL
|
||||
fi
|
||||
echo TTY_ALPHABET_END
|
||||
else
|
||||
echo DEVMEM_MISSING_OR_NO_DEV
|
||||
fi
|
||||
echo E2E_END
|
||||
sync
|
||||
poweroff -f
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
#
|
||||
# Best-effort cleanup helper for ./fake_pci_sriov.ko.
|
||||
#
|
||||
# Purpose: return the host to a clean pci-sim state by unbinding fake VFs,
|
||||
# disabling SR-IOV on fake PFs, removing fake PF devices, and optionally
|
||||
# unloading the module. DevStack and smoke tests use this before/after runs so
|
||||
# repeated validation starts from a deterministic topology.
|
||||
#
|
||||
# Prerequisites: sudo without a password and sysfs PCI device access.
|
||||
#
|
||||
# Important environment variables:
|
||||
# - VENDOR/PF_DEVICE/VF_DEVICE: fake PCI IDs, defaults 0x1d55/0x1000/0x1001
|
||||
# - MODULE_NAME: module to unload, default fake_pci_sriov
|
||||
# - RMMOD: unload MODULE_NAME when set to 1, default 1
|
||||
# - SUDO: privilege wrapper, default "sudo -n"
|
||||
# - LOG: optional log file; stdout is used when LOG is unset
|
||||
#
|
||||
# Exit status is zero when cleanup commands either succeed or find nothing to
|
||||
# clean. Example:
|
||||
# LOG=/tmp/fake_pci_cleanup.log bash pci-sim/cleanup_fake_pci_sriov.sh
|
||||
set -euo pipefail
|
||||
|
||||
VENDOR=${VENDOR:-0x1d55}
|
||||
PF_DEVICE=${PF_DEVICE:-0x1000}
|
||||
VF_DEVICE=${VF_DEVICE:-0x1001}
|
||||
MODULE_NAME=${MODULE_NAME:-fake_pci_sriov}
|
||||
RMMOD=${RMMOD:-1}
|
||||
SUDO=${SUDO:-sudo -n}
|
||||
# Default: log to stdout so callers (DevStack, CI) capture output through
|
||||
# their own logging. Set LOG=/path/to/file to additionally tee to a file.
|
||||
if [[ -n "${LOG:-}" ]]; then
|
||||
: > "$LOG"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
fi
|
||||
|
||||
msg() { echo "== $* =="; }
|
||||
|
||||
sysfs_write() {
|
||||
local value=$1 path=$2
|
||||
|
||||
[ -e "$path" ] || return 0
|
||||
printf '%s\n' "$value" | $SUDO tee "$path" >/dev/null
|
||||
}
|
||||
|
||||
find_fake_devs() {
|
||||
local device=$1 d
|
||||
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
[ "$(cat "$d/vendor" 2>/dev/null || true)" = "$VENDOR" ] || continue
|
||||
[ "$(cat "$d/device" 2>/dev/null || true)" = "$device" ] || continue
|
||||
basename "$d"
|
||||
done | sort
|
||||
}
|
||||
|
||||
unbind_dev() {
|
||||
local dev=$1 base=/sys/bus/pci/devices/$1
|
||||
|
||||
if [ -e "$base/driver/unbind" ]; then
|
||||
msg "unbind $dev from $(basename "$(readlink -f "$base/driver")")"
|
||||
sysfs_write "$dev" "$base/driver/unbind"
|
||||
fi
|
||||
|
||||
sysfs_write '' "$base/driver_override"
|
||||
}
|
||||
|
||||
msg "preflight"
|
||||
$SUDO true
|
||||
|
||||
msg "unbind fake VFs"
|
||||
for vf in $(find_fake_devs "$VF_DEVICE"); do
|
||||
unbind_dev "$vf" || true
|
||||
done
|
||||
|
||||
msg "disable SR-IOV on fake PFs"
|
||||
for pf in $(find_fake_devs "$PF_DEVICE"); do
|
||||
sysfs_write 0 "/sys/bus/pci/devices/$pf/sriov_numvfs" || true
|
||||
done
|
||||
|
||||
msg "remove fake PFs"
|
||||
for pf in $(find_fake_devs "$PF_DEVICE"); do
|
||||
unbind_dev "$pf" || true
|
||||
sysfs_write 1 "/sys/bus/pci/devices/$pf/remove" || true
|
||||
done
|
||||
|
||||
if [ "$RMMOD" = 1 ] && grep -q "^$MODULE_NAME " /proc/modules; then
|
||||
msg "rmmod $MODULE_NAME"
|
||||
$SUDO rmmod "$MODULE_NAME"
|
||||
fi
|
||||
|
||||
msg "remaining fake devices"
|
||||
find_fake_devs "$PF_DEVICE" || true
|
||||
find_fake_devs "$VF_DEVICE" || true
|
||||
msg PASS
|
||||
@@ -0,0 +1,187 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* Fake PCI SR-IOV VFIO test fixture -- shared internal header
|
||||
*
|
||||
* All compilation units in the fake_pci_sriov multi-file module include this
|
||||
* header. It centralises struct definitions, constants, and cross-file
|
||||
* declarations so individual source files stay focused on a single subsystem.
|
||||
*/
|
||||
|
||||
#ifndef FAKE_PCI_SRIOV_H
|
||||
#define FAKE_PCI_SRIOV_H
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/pci_regs.h>
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/idr.h>
|
||||
#include <linux/xarray.h>
|
||||
#include <linux/serial_reg.h>
|
||||
#include <linux/tty.h>
|
||||
#include <linux/tty_flip.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <linux/vfio_pci_core.h>
|
||||
#include <linux/string.h>
|
||||
#include <linux/unaligned.h>
|
||||
#include <linux/ioport.h>
|
||||
#include <linux/list.h>
|
||||
|
||||
#include "fake_pci_sriov_compat.h"
|
||||
|
||||
#ifdef CONFIG_X86
|
||||
#include <asm/pci.h>
|
||||
#elif defined(CONFIG_ACPI)
|
||||
#include <linux/pci-ecam.h>
|
||||
#endif
|
||||
|
||||
/* =========================================================================
|
||||
* Device identification
|
||||
* ========================================================================= */
|
||||
|
||||
#define FAKE_PCI_VENDOR_ID 0x1d55
|
||||
#define FAKE_PCI_PF_DEVICE_ID 0x1000
|
||||
#define FAKE_PCI_VF_DEVICE_ID 0x1001
|
||||
#define FAKE_PCI_SERIAL_CLASS 0x070002 /* Serial controller, 16550 */
|
||||
#define FAKE_PCI_VENDOR_CLASS 0xff0000
|
||||
#define FAKE_PCI_SUBSYS_VENDOR FAKE_PCI_VENDOR_ID
|
||||
#define FAKE_PCI_SUBSYS_ID FAKE_PCI_PF_DEVICE_ID
|
||||
|
||||
/* =========================================================================
|
||||
* SR-IOV / BAR configuration
|
||||
* ========================================================================= */
|
||||
|
||||
#define MAX_VFS 7
|
||||
#define FAKE_PCI_MAX_HOSTS 16
|
||||
#define SRIOV_CAP_OFFSET 0x100
|
||||
#define PCIE_CAP_OFFSET 0x40
|
||||
|
||||
#define BAR0_SIZE 0x1000
|
||||
#define FAKE_PCI_MEM_BASE 0xd0000000ULL
|
||||
#define FAKE_PCI_MEM_64_BASE 0x100000000ULL
|
||||
#define FAKE_PCI_MEM_STRIDE 0x00100000
|
||||
#define FAKE_PCI_MEM_MIN_SIZE (BAR0_SIZE * (MAX_VFS + 1))
|
||||
#define FAKE_PCI_BAR_FLAGS PCI_BASE_ADDRESS_MEM_TYPE_64
|
||||
#define PCI_SIM_VFIO_BAR0_SIZE 0x40000
|
||||
#define PCI_SIM_SGI_IOC3_UART_OFFSET 0x20178
|
||||
|
||||
/* =========================================================================
|
||||
* Host-side TTY loopback
|
||||
* ========================================================================= */
|
||||
|
||||
#define PCI_SIM_TTY_NAME "ttyPCI_SIM"
|
||||
#define PCI_SIM_MAX_TTYS 256
|
||||
#define PCI_SIM_UART_FIFO_SIZE 4096
|
||||
#define PCI_SIM_UART_CHUNK 256
|
||||
|
||||
/* =========================================================================
|
||||
* Data structures
|
||||
* ========================================================================= */
|
||||
|
||||
struct fake_pci_device {
|
||||
u8 config_space[4096];
|
||||
bool present;
|
||||
bool is_vf;
|
||||
int vf_index;
|
||||
};
|
||||
|
||||
struct fake_pci_host {
|
||||
struct list_head list;
|
||||
struct pci_host_bridge *bridge;
|
||||
struct platform_device *pdev;
|
||||
#ifdef CONFIG_X86
|
||||
struct pci_sysdata sysdata;
|
||||
#elif defined(CONFIG_ACPI)
|
||||
struct pci_config_window sysdata;
|
||||
#endif
|
||||
struct resource bus_resource;
|
||||
struct resource mem_resource;
|
||||
bool mem_resource_registered;
|
||||
struct fake_pci_device pf;
|
||||
struct fake_pci_device vfs[MAX_VFS];
|
||||
int num_vfs_enabled;
|
||||
int domain_nr;
|
||||
struct mutex lock;
|
||||
};
|
||||
|
||||
struct pci_sim_uart {
|
||||
spinlock_t lock;
|
||||
u8 regs[8];
|
||||
u8 fifo[PCI_SIM_UART_FIFO_SIZE];
|
||||
unsigned int head;
|
||||
unsigned int tail;
|
||||
unsigned int count;
|
||||
bool dlab;
|
||||
bool overrun;
|
||||
u16 divisor;
|
||||
u8 fcr;
|
||||
u8 intr_trigger_level;
|
||||
};
|
||||
|
||||
struct pci_sim_vf_tty {
|
||||
struct pci_dev *pdev;
|
||||
struct tty_port port;
|
||||
struct mutex state_lock;
|
||||
struct pci_sim_uart uart;
|
||||
int id;
|
||||
bool dead;
|
||||
};
|
||||
|
||||
struct pci_sim_vfio_vf {
|
||||
struct vfio_pci_core_device core;
|
||||
struct mutex lock;
|
||||
struct pci_sim_uart uart;
|
||||
};
|
||||
|
||||
/* =========================================================================
|
||||
* Cross-file globals (defined in the owning .c, declared extern here)
|
||||
* ========================================================================= */
|
||||
|
||||
/* fake_pci_sriov_core.c */
|
||||
extern struct list_head fake_hosts;
|
||||
extern struct mutex fake_hosts_lock;
|
||||
|
||||
/* Module parameters that multiple files read */
|
||||
extern bool vf_serial_class;
|
||||
extern bool vfio_guest_8250_compat;
|
||||
extern bool vfio_uart_trace;
|
||||
extern unsigned long mem_stride;
|
||||
|
||||
/* fake_pci_sriov_iommu.c */
|
||||
extern struct iommu_device fake_iommu_dev;
|
||||
extern struct platform_device *fake_iommu_pdev;
|
||||
|
||||
/* =========================================================================
|
||||
* Cross-file function declarations
|
||||
* ========================================================================= */
|
||||
|
||||
/* fake_pci_sriov_cfg.c */
|
||||
void init_pf_config_space(struct fake_pci_device *dev);
|
||||
struct fake_pci_host *fake_pci_host_from_domain(int domain);
|
||||
|
||||
/* fake_pci_sriov_uart.c */
|
||||
int pci_sim_tty_register_driver(void);
|
||||
void pci_sim_tty_unregister_driver(void);
|
||||
|
||||
/* UART primitives also used by vfio.c */
|
||||
void pci_sim_uart_reset(struct pci_sim_uart *uart);
|
||||
void pci_sim_uart_init(struct pci_sim_uart *uart);
|
||||
size_t pci_sim_uart_write_data(struct pci_sim_uart *uart, const u8 *buf, size_t len);
|
||||
unsigned int pci_sim_uart_write_room(struct pci_sim_uart *uart);
|
||||
unsigned int pci_sim_uart_chars_in_buffer(struct pci_sim_uart *uart);
|
||||
u8 pci_sim_uart_lsr(struct pci_sim_uart *uart);
|
||||
u8 pci_sim_uart_read_data(struct pci_sim_uart *uart);
|
||||
u8 pci_sim_uart_read_reg(struct pci_sim_uart *uart, u8 reg);
|
||||
void pci_sim_uart_write_reg(struct pci_sim_uart *uart, u8 reg, u8 val);
|
||||
|
||||
/* fake_pci_sriov_iommu.c (ops table used by core init) */
|
||||
extern const struct iommu_ops fake_iommu_ops;
|
||||
|
||||
/* fake_pci_sriov_cfg.c (pci_ops used by core host-probe) */
|
||||
extern struct pci_ops fake_pci_ops;
|
||||
|
||||
#endif /* FAKE_PCI_SRIOV_H */
|
||||
@@ -0,0 +1,508 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Fake PCI SR-IOV VFIO test fixture -- PCI config space and SR-IOV management
|
||||
*
|
||||
* Implements the fake pci_ops config-space read/write paths, PCI capability
|
||||
* initialisation (PCIe, SR-IOV), and the PF driver that owns sriov_configure.
|
||||
*/
|
||||
|
||||
#include "fake_pci_sriov.h"
|
||||
|
||||
/* =========================================================================
|
||||
* Config-space byte helpers
|
||||
* ========================================================================= */
|
||||
|
||||
static u16 fake_cfg_read16(const u8 *config, int where)
|
||||
{
|
||||
return get_unaligned_le16(config + where);
|
||||
}
|
||||
|
||||
static u32 fake_cfg_read32(const u8 *config, int where)
|
||||
{
|
||||
return get_unaligned_le32(config + where);
|
||||
}
|
||||
|
||||
static void fake_cfg_write16(u8 *config, int where, u16 val)
|
||||
{
|
||||
put_unaligned_le16(val, config + where);
|
||||
}
|
||||
|
||||
static void fake_cfg_write32(u8 *config, int where, u32 val)
|
||||
{
|
||||
put_unaligned_le32(val, config + where);
|
||||
}
|
||||
|
||||
static int fake_cfg_read(const u8 *config, int where, int size, u32 *val)
|
||||
{
|
||||
switch (size) {
|
||||
case 1:
|
||||
*val = config[where];
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
case 2:
|
||||
*val = fake_cfg_read16(config, where);
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
case 4:
|
||||
*val = fake_cfg_read32(config, where);
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
default:
|
||||
*val = ~0;
|
||||
return PCIBIOS_BAD_REGISTER_NUMBER;
|
||||
}
|
||||
}
|
||||
|
||||
static int fake_cfg_write(u8 *config, int where, int size, u32 val)
|
||||
{
|
||||
switch (size) {
|
||||
case 1:
|
||||
config[where] = val;
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
case 2:
|
||||
fake_cfg_write16(config, where, val);
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
case 4:
|
||||
fake_cfg_write32(config, where, val);
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
default:
|
||||
return PCIBIOS_BAD_REGISTER_NUMBER;
|
||||
}
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* Host-lookup helpers (used by iommu.c and pci_ops below)
|
||||
* ========================================================================= */
|
||||
|
||||
struct fake_pci_host *fake_pci_host_from_domain(int domain)
|
||||
{
|
||||
struct fake_pci_host *host;
|
||||
|
||||
lockdep_assert_held(&fake_hosts_lock);
|
||||
|
||||
list_for_each_entry(host, &fake_hosts, list) {
|
||||
if (domain == host->domain_nr)
|
||||
return host;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* PCI capability initialisation
|
||||
* ========================================================================= */
|
||||
|
||||
static void fake_pci_set_class(u8 *config, u32 class)
|
||||
{
|
||||
config[PCI_CLASS_PROG] = class & 0xff;
|
||||
config[PCI_CLASS_DEVICE] = (class >> 8) & 0xff;
|
||||
config[PCI_CLASS_DEVICE + 1] = (class >> 16) & 0xff;
|
||||
}
|
||||
|
||||
static void init_pcie_capability(u8 *config, bool is_pf)
|
||||
{
|
||||
u8 *cap = &config[PCIE_CAP_OFFSET];
|
||||
u32 link_cap;
|
||||
u16 exp_flags, link_status;
|
||||
|
||||
cap[PCI_CAP_LIST_ID] = PCI_CAP_ID_EXP;
|
||||
cap[PCI_CAP_LIST_NEXT] = 0;
|
||||
|
||||
exp_flags = 2 | (PCI_EXP_TYPE_ENDPOINT << 4);
|
||||
fake_cfg_write16(cap, PCI_EXP_FLAGS, exp_flags);
|
||||
|
||||
fake_cfg_write32(cap, PCI_EXP_DEVCAP, PCI_EXP_DEVCAP_FLR);
|
||||
fake_cfg_write16(cap, PCI_EXP_DEVCTL, 0);
|
||||
fake_cfg_write16(cap, PCI_EXP_DEVSTA, 0);
|
||||
|
||||
link_cap = PCI_EXP_LNKCAP_SLS_2_5GB | (1 << 4);
|
||||
fake_cfg_write32(cap, PCI_EXP_LNKCAP, link_cap);
|
||||
|
||||
link_status = PCI_EXP_LNKSTA_CLS_2_5GB | PCI_EXP_LNKSTA_NLW_X1;
|
||||
fake_cfg_write16(cap, PCI_EXP_LNKSTA, link_status);
|
||||
}
|
||||
|
||||
static void init_sriov_capability(u8 *config)
|
||||
{
|
||||
u8 *cap = &config[SRIOV_CAP_OFFSET];
|
||||
|
||||
fake_cfg_write16(cap, 0, PCI_EXT_CAP_ID_SRIOV);
|
||||
fake_cfg_write16(cap, 2, 0);
|
||||
fake_cfg_write32(cap, PCI_SRIOV_CAP, 0);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_CTRL, 0);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_STATUS, 0);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_INITIAL_VF, MAX_VFS);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_TOTAL_VF, MAX_VFS);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_NUM_VF, 0);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_FUNC_LINK, 0);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_VF_OFFSET, 1);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_VF_STRIDE, 1);
|
||||
fake_cfg_write16(cap, PCI_SRIOV_VF_DID, FAKE_PCI_VF_DEVICE_ID);
|
||||
fake_cfg_write32(cap, PCI_SRIOV_SUP_PGSIZE, 1);
|
||||
fake_cfg_write32(cap, PCI_SRIOV_SYS_PGSIZE, 1);
|
||||
fake_cfg_write32(cap, PCI_SRIOV_BAR, FAKE_PCI_BAR_FLAGS);
|
||||
fake_cfg_write32(cap, PCI_SRIOV_BAR + 4, 0);
|
||||
}
|
||||
|
||||
static int fake_pci_write_bar(u8 *config, int where, u32 val, int base)
|
||||
{
|
||||
int bar_num = (where - base) / 4;
|
||||
|
||||
if (bar_num == 0) {
|
||||
if (val == 0xffffffff) {
|
||||
fake_cfg_write32(config, where,
|
||||
~(BAR0_SIZE - 1) | FAKE_PCI_BAR_FLAGS);
|
||||
} else {
|
||||
fake_cfg_write32(config, where,
|
||||
(val & ~(BAR0_SIZE - 1)) |
|
||||
FAKE_PCI_BAR_FLAGS);
|
||||
}
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
}
|
||||
|
||||
if (bar_num == 1) {
|
||||
fake_cfg_write32(config, where,
|
||||
val == 0xffffffff ? 0xffffffff : val);
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
}
|
||||
|
||||
fake_cfg_write32(config, where, 0);
|
||||
return PCIBIOS_SUCCESSFUL;
|
||||
}
|
||||
|
||||
void init_pf_config_space(struct fake_pci_device *dev)
|
||||
{
|
||||
u8 *config = dev->config_space;
|
||||
|
||||
memset(config, 0, sizeof(dev->config_space));
|
||||
dev->present = true;
|
||||
dev->is_vf = false;
|
||||
|
||||
fake_cfg_write16(config, PCI_VENDOR_ID, FAKE_PCI_VENDOR_ID);
|
||||
fake_cfg_write16(config, PCI_DEVICE_ID, FAKE_PCI_PF_DEVICE_ID);
|
||||
fake_cfg_write16(config, PCI_COMMAND,
|
||||
PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
|
||||
fake_cfg_write16(config, PCI_STATUS, PCI_STATUS_CAP_LIST);
|
||||
|
||||
config[PCI_REVISION_ID] = 0x01;
|
||||
fake_pci_set_class(config, FAKE_PCI_VENDOR_CLASS);
|
||||
config[PCI_CACHE_LINE_SIZE] = 64 / 4;
|
||||
config[PCI_HEADER_TYPE] = PCI_HEADER_TYPE_NORMAL | PCI_HEADER_TYPE_MFD;
|
||||
|
||||
fake_cfg_write32(config, PCI_BASE_ADDRESS_0, FAKE_PCI_BAR_FLAGS);
|
||||
fake_cfg_write32(config, PCI_BASE_ADDRESS_1, 0);
|
||||
fake_cfg_write16(config, PCI_SUBSYSTEM_VENDOR_ID, FAKE_PCI_SUBSYS_VENDOR);
|
||||
fake_cfg_write16(config, PCI_SUBSYSTEM_ID, FAKE_PCI_SUBSYS_ID);
|
||||
|
||||
config[PCI_CAPABILITY_LIST] = PCIE_CAP_OFFSET;
|
||||
config[PCI_INTERRUPT_LINE] = 0;
|
||||
config[PCI_INTERRUPT_PIN] = 1;
|
||||
|
||||
init_pcie_capability(config, true);
|
||||
init_sriov_capability(config);
|
||||
}
|
||||
|
||||
static void init_vf_config_space(struct fake_pci_device *dev, int vf_index)
|
||||
{
|
||||
u8 *config = dev->config_space;
|
||||
|
||||
memset(config, 0, sizeof(dev->config_space));
|
||||
dev->present = true;
|
||||
dev->is_vf = true;
|
||||
dev->vf_index = vf_index;
|
||||
|
||||
fake_cfg_write16(config, PCI_VENDOR_ID, FAKE_PCI_VENDOR_ID);
|
||||
fake_cfg_write16(config, PCI_DEVICE_ID, FAKE_PCI_VF_DEVICE_ID);
|
||||
fake_cfg_write16(config, PCI_COMMAND,
|
||||
PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
|
||||
fake_cfg_write16(config, PCI_STATUS, PCI_STATUS_CAP_LIST);
|
||||
|
||||
config[PCI_REVISION_ID] = 0x01;
|
||||
fake_pci_set_class(config, vf_serial_class ? FAKE_PCI_SERIAL_CLASS :
|
||||
FAKE_PCI_VENDOR_CLASS);
|
||||
config[PCI_CACHE_LINE_SIZE] = 64 / 4;
|
||||
config[PCI_HEADER_TYPE] = PCI_HEADER_TYPE_NORMAL;
|
||||
|
||||
fake_cfg_write32(config, PCI_BASE_ADDRESS_0, FAKE_PCI_BAR_FLAGS);
|
||||
fake_cfg_write32(config, PCI_BASE_ADDRESS_1, 0);
|
||||
fake_cfg_write16(config, PCI_SUBSYSTEM_VENDOR_ID, FAKE_PCI_SUBSYS_VENDOR);
|
||||
fake_cfg_write16(config, PCI_SUBSYSTEM_ID, FAKE_PCI_SUBSYS_ID);
|
||||
|
||||
config[PCI_CAPABILITY_LIST] = PCIE_CAP_OFFSET;
|
||||
|
||||
init_pcie_capability(config, false);
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* pci_ops: config-space read/write
|
||||
* ========================================================================= */
|
||||
|
||||
static struct fake_pci_device *get_fake_device(struct fake_pci_host *host,
|
||||
unsigned int devfn)
|
||||
{
|
||||
int slot = PCI_SLOT(devfn);
|
||||
int func = PCI_FUNC(devfn);
|
||||
|
||||
if (slot != 0)
|
||||
return NULL;
|
||||
|
||||
if (func == 0)
|
||||
return &host->pf;
|
||||
|
||||
if (func >= 1 && func <= MAX_VFS) {
|
||||
int vf_idx = func - 1;
|
||||
|
||||
if (vf_idx < host->num_vfs_enabled)
|
||||
return &host->vfs[vf_idx];
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int fake_pci_read_config(struct pci_bus *bus, unsigned int devfn,
|
||||
int where, int size, u32 *val)
|
||||
{
|
||||
struct fake_pci_host *host;
|
||||
struct fake_pci_device *dev;
|
||||
int ret;
|
||||
|
||||
if (bus->number != 0) {
|
||||
*val = ~0;
|
||||
return PCIBIOS_DEVICE_NOT_FOUND;
|
||||
}
|
||||
|
||||
mutex_lock(&fake_hosts_lock);
|
||||
host = fake_pci_host_from_domain(pci_domain_nr(bus));
|
||||
if (!host) {
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
*val = ~0;
|
||||
return PCIBIOS_DEVICE_NOT_FOUND;
|
||||
}
|
||||
|
||||
dev = get_fake_device(host, devfn);
|
||||
if (!dev || !dev->present) {
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
*val = ~0;
|
||||
return PCIBIOS_DEVICE_NOT_FOUND;
|
||||
}
|
||||
|
||||
if (where + size > sizeof(dev->config_space)) {
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
*val = ~0;
|
||||
return PCIBIOS_BAD_REGISTER_NUMBER;
|
||||
}
|
||||
|
||||
ret = fake_cfg_read(dev->config_space, where, size, val);
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void handle_sriov_numvfs_write(struct fake_pci_host *host, u16 num_vfs);
|
||||
|
||||
static bool fake_pci_is_sriov_cfg(int where)
|
||||
{
|
||||
return where >= SRIOV_CAP_OFFSET &&
|
||||
where < SRIOV_CAP_OFFSET + PCI_SRIOV_BAR +
|
||||
PCI_SRIOV_NUM_BARS * 4;
|
||||
}
|
||||
|
||||
static int fake_pci_write_config(struct pci_bus *bus, unsigned int devfn,
|
||||
int where, int size, u32 val)
|
||||
{
|
||||
struct fake_pci_host *host;
|
||||
struct fake_pci_device *dev;
|
||||
u8 *config;
|
||||
int ret;
|
||||
|
||||
if (bus->number != 0)
|
||||
return PCIBIOS_DEVICE_NOT_FOUND;
|
||||
|
||||
mutex_lock(&fake_hosts_lock);
|
||||
host = fake_pci_host_from_domain(pci_domain_nr(bus));
|
||||
if (!host) {
|
||||
ret = PCIBIOS_DEVICE_NOT_FOUND;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
dev = get_fake_device(host, devfn);
|
||||
if (!dev || !dev->present) {
|
||||
ret = PCIBIOS_DEVICE_NOT_FOUND;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
if (where + size > sizeof(dev->config_space)) {
|
||||
ret = PCIBIOS_BAD_REGISTER_NUMBER;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
config = dev->config_space;
|
||||
|
||||
if (size == 4 && where >= PCI_BASE_ADDRESS_0 &&
|
||||
where < PCI_BASE_ADDRESS_5 + 4) {
|
||||
ret = fake_pci_write_bar(config, where, val, PCI_BASE_ADDRESS_0);
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
if (size == 4 && where == PCI_ROM_ADDRESS) {
|
||||
fake_cfg_write32(config, PCI_ROM_ADDRESS, 0);
|
||||
ret = PCIBIOS_SUCCESSFUL;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
if (size == 4 && !dev->is_vf &&
|
||||
where >= SRIOV_CAP_OFFSET + PCI_SRIOV_BAR &&
|
||||
where < SRIOV_CAP_OFFSET + PCI_SRIOV_BAR +
|
||||
PCI_SRIOV_NUM_BARS * 4) {
|
||||
ret = fake_pci_write_bar(config, where, val,
|
||||
SRIOV_CAP_OFFSET + PCI_SRIOV_BAR);
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
if (!dev->is_vf && fake_pci_is_sriov_cfg(where)) {
|
||||
ret = fake_cfg_write(config, where, size, val);
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
switch (where) {
|
||||
case PCI_VENDOR_ID:
|
||||
case PCI_DEVICE_ID:
|
||||
case PCI_REVISION_ID:
|
||||
case PCI_CLASS_PROG:
|
||||
case PCI_CLASS_DEVICE:
|
||||
case PCI_HEADER_TYPE:
|
||||
case PCI_SUBSYSTEM_VENDOR_ID:
|
||||
case PCI_SUBSYSTEM_ID:
|
||||
case PCI_CAPABILITY_LIST:
|
||||
ret = PCIBIOS_SUCCESSFUL;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
ret = fake_cfg_write(config, where, size, val);
|
||||
|
||||
out_unlock:
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
struct pci_ops fake_pci_ops = {
|
||||
.read = fake_pci_read_config,
|
||||
.write = fake_pci_write_config,
|
||||
};
|
||||
|
||||
/* =========================================================================
|
||||
* SR-IOV VF enable/disable
|
||||
* ========================================================================= */
|
||||
|
||||
static void handle_sriov_numvfs_write(struct fake_pci_host *host, u16 num_vfs)
|
||||
{
|
||||
u8 *pf_config = host->pf.config_space;
|
||||
int i;
|
||||
|
||||
mutex_lock(&host->lock);
|
||||
|
||||
if (num_vfs > MAX_VFS)
|
||||
num_vfs = MAX_VFS;
|
||||
|
||||
pr_info("fake_pci: Setting NumVFs from %d to %d\n",
|
||||
host->num_vfs_enabled, num_vfs);
|
||||
|
||||
for (i = num_vfs; i < host->num_vfs_enabled; i++) {
|
||||
host->vfs[i].present = false;
|
||||
pr_info("fake_pci: Disabled VF%d\n", i);
|
||||
}
|
||||
|
||||
for (i = host->num_vfs_enabled; i < num_vfs; i++) {
|
||||
init_vf_config_space(&host->vfs[i], i);
|
||||
pr_info("fake_pci: Enabled VF%d\n", i);
|
||||
}
|
||||
|
||||
host->num_vfs_enabled = num_vfs;
|
||||
fake_cfg_write16(pf_config, SRIOV_CAP_OFFSET + PCI_SRIOV_NUM_VF, num_vfs);
|
||||
|
||||
mutex_unlock(&host->lock);
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* PF driver
|
||||
* ========================================================================= */
|
||||
|
||||
static struct fake_pci_host *fake_pci_host_from_pdev(struct pci_dev *pdev)
|
||||
{
|
||||
struct fake_pci_host *host;
|
||||
|
||||
mutex_lock(&fake_hosts_lock);
|
||||
host = fake_pci_host_from_domain(pci_domain_nr(pdev->bus));
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
|
||||
return host;
|
||||
}
|
||||
|
||||
static int fake_pci_pf_probe(struct pci_dev *pdev,
|
||||
const struct pci_device_id *id)
|
||||
{
|
||||
struct fake_pci_host *host = fake_pci_host_from_pdev(pdev);
|
||||
|
||||
if (!host)
|
||||
return -ENODEV;
|
||||
|
||||
pci_set_drvdata(pdev, host);
|
||||
pci_info(pdev, "fake_pci: PF probed\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void fake_pci_pf_remove(struct pci_dev *pdev)
|
||||
{
|
||||
struct fake_pci_host *host = pci_get_drvdata(pdev);
|
||||
|
||||
if (pci_num_vf(pdev)) {
|
||||
pci_info(pdev, "fake_pci: disabling VFs before PF removal\n");
|
||||
pci_disable_sriov(pdev);
|
||||
if (host)
|
||||
handle_sriov_numvfs_write(host, 0);
|
||||
}
|
||||
|
||||
pci_set_drvdata(pdev, NULL);
|
||||
pci_info(pdev, "fake_pci: PF removed\n");
|
||||
}
|
||||
|
||||
static int fake_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
|
||||
{
|
||||
struct fake_pci_host *host = pci_get_drvdata(pdev);
|
||||
int err;
|
||||
|
||||
if (!host)
|
||||
return -ENODEV;
|
||||
|
||||
if (num_vfs < 0 || num_vfs > MAX_VFS)
|
||||
return -EINVAL;
|
||||
|
||||
if (!num_vfs) {
|
||||
pci_disable_sriov(pdev);
|
||||
handle_sriov_numvfs_write(host, 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (host->num_vfs_enabled)
|
||||
return -EBUSY;
|
||||
|
||||
handle_sriov_numvfs_write(host, num_vfs);
|
||||
|
||||
err = pci_enable_sriov(pdev, num_vfs);
|
||||
if (err) {
|
||||
handle_sriov_numvfs_write(host, 0);
|
||||
return err;
|
||||
}
|
||||
|
||||
return num_vfs;
|
||||
}
|
||||
|
||||
static const struct pci_device_id fake_pci_pf_ids[] = {
|
||||
{ PCI_DEVICE(FAKE_PCI_VENDOR_ID, FAKE_PCI_PF_DEVICE_ID) },
|
||||
{ }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(pci, fake_pci_pf_ids);
|
||||
|
||||
struct pci_driver fake_pci_pf_driver = {
|
||||
.name = "fake_pci_sriov_pf",
|
||||
.id_table = fake_pci_pf_ids,
|
||||
.probe = fake_pci_pf_probe,
|
||||
.remove = fake_pci_pf_remove,
|
||||
.sriov_configure = fake_pci_sriov_configure,
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef FAKE_PCI_SRIOV_COMPAT_H
|
||||
#define FAKE_PCI_SRIOV_COMPAT_H
|
||||
|
||||
#include <linux/version.h>
|
||||
|
||||
/*
|
||||
* Keep kernel API compatibility decisions in one place. The module is
|
||||
* developed against newer upstream/Ubuntu kernels, while Debian 13 currently
|
||||
* ships a 6.12 kernel whose IOMMU and VFIO callback tables predate some of the
|
||||
* newer PASID/HWPT and VFIO region-info hooks.
|
||||
*
|
||||
* These version gates intentionally select the conservative 6.12-compatible
|
||||
* API for older kernels. Newer kernels keep using the richer callbacks.
|
||||
*/
|
||||
|
||||
/* iommu_domain_ops.attach_dev gained the previous-domain argument later. */
|
||||
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
|
||||
#define PCI_SIM_IOMMU_ATTACH_HAS_OLD_DOMAIN 1
|
||||
#else
|
||||
#define PCI_SIM_IOMMU_ATTACH_HAS_OLD_DOMAIN 0
|
||||
#endif
|
||||
|
||||
/* iommu_ops.domain_alloc_paging_flags and IOMMU_HWPT_ALLOC_PASID are newer. */
|
||||
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
|
||||
#define PCI_SIM_IOMMU_HAS_DOMAIN_ALLOC_PAGING_FLAGS 1
|
||||
#else
|
||||
#define PCI_SIM_IOMMU_HAS_DOMAIN_ALLOC_PAGING_FLAGS 0
|
||||
#endif
|
||||
|
||||
/* VFIO grew optional region-info and PASID/token callbacks after 6.12. */
|
||||
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
|
||||
#define PCI_SIM_VFIO_HAS_GET_REGION_INFO_CAPS 1
|
||||
#define PCI_SIM_VFIO_HAS_MATCH_TOKEN_UUID 1
|
||||
#define PCI_SIM_VFIO_HAS_PASID_IOAS 1
|
||||
#else
|
||||
#define PCI_SIM_VFIO_HAS_GET_REGION_INFO_CAPS 0
|
||||
#define PCI_SIM_VFIO_HAS_MATCH_TOKEN_UUID 0
|
||||
#define PCI_SIM_VFIO_HAS_PASID_IOAS 0
|
||||
#endif
|
||||
|
||||
#endif /* FAKE_PCI_SRIOV_COMPAT_H */
|
||||
@@ -0,0 +1,436 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Fake PCI SR-IOV VFIO test fixture -- module lifecycle and host bridge setup
|
||||
*
|
||||
* This file owns the module parameters, the global host list, the PCI host
|
||||
* bridge probe/remove lifecycle, and the module init/exit entry points.
|
||||
* It coordinates the other subsystems (iommu, cfg, uart, vfio) by calling
|
||||
* their registration helpers in the correct order.
|
||||
*/
|
||||
|
||||
#include "fake_pci_sriov.h"
|
||||
|
||||
/* =========================================================================
|
||||
* Module parameters
|
||||
* ========================================================================= */
|
||||
|
||||
bool vf_serial_class;
|
||||
module_param(vf_serial_class, bool, 0644);
|
||||
MODULE_PARM_DESC(vf_serial_class,
|
||||
"Expose host-visible VFs as PCI serial/16550 class devices instead of vendor-specific");
|
||||
|
||||
bool vfio_guest_8250_compat = true;
|
||||
module_param(vfio_guest_8250_compat, bool, 0644);
|
||||
MODULE_PARM_DESC(vfio_guest_8250_compat,
|
||||
"Overlay VFIO guest config space with an 8250_pci-compatible SGI IOC3 serial identity");
|
||||
|
||||
bool vfio_uart_trace;
|
||||
module_param(vfio_uart_trace, bool, 0644);
|
||||
MODULE_PARM_DESC(vfio_uart_trace,
|
||||
"Trace VFIO BAR0 UART register accesses");
|
||||
|
||||
static int fake_intx_irq;
|
||||
module_param(fake_intx_irq, int, 0644);
|
||||
MODULE_PARM_DESC(fake_intx_irq,
|
||||
"Optional host IRQ number to report for fake PCI INTx routing (0 disables)");
|
||||
|
||||
static unsigned int num_pfs = 1;
|
||||
module_param(num_pfs, uint, 0444);
|
||||
MODULE_PARM_DESC(num_pfs,
|
||||
"Number of fake SR-IOV PFs to create");
|
||||
|
||||
static unsigned long mem_base;
|
||||
module_param(mem_base, ulong, 0444);
|
||||
MODULE_PARM_DESC(mem_base,
|
||||
"Fixed fake PCI MMIO window base address (0 chooses automatically)");
|
||||
|
||||
unsigned long mem_stride = FAKE_PCI_MEM_STRIDE;
|
||||
module_param(mem_stride, ulong, 0444);
|
||||
MODULE_PARM_DESC(mem_stride,
|
||||
"Size and alignment of each fake PCI host MMIO window");
|
||||
|
||||
/* =========================================================================
|
||||
* Global host list (also used by cfg.c and iommu.c)
|
||||
* ========================================================================= */
|
||||
|
||||
LIST_HEAD(fake_hosts);
|
||||
DEFINE_MUTEX(fake_hosts_lock);
|
||||
|
||||
/* =========================================================================
|
||||
* PCI host bridge helpers
|
||||
* ========================================================================= */
|
||||
|
||||
static void *fake_pci_host_sysdata(struct fake_pci_host *host)
|
||||
{
|
||||
#ifdef CONFIG_X86
|
||||
return &host->sysdata;
|
||||
#elif defined(CONFIG_ACPI)
|
||||
return &host->sysdata;
|
||||
#else
|
||||
return NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void fake_pci_host_init_sysdata(struct fake_pci_host *host)
|
||||
{
|
||||
#ifdef CONFIG_X86
|
||||
host->sysdata.domain = host->domain_nr;
|
||||
host->sysdata.node = NUMA_NO_NODE;
|
||||
#elif defined(CONFIG_ACPI)
|
||||
host->sysdata.priv = host;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void fake_pci_release_host_bridge(struct pci_host_bridge *bridge)
|
||||
{
|
||||
pr_info("fake_pci: releasing host bridge\n");
|
||||
}
|
||||
|
||||
static int fake_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
|
||||
{
|
||||
return fake_intx_irq ?: -1;
|
||||
}
|
||||
|
||||
static void fake_pci_host_release_mem_resource(struct fake_pci_host *host)
|
||||
{
|
||||
if (!host->mem_resource_registered)
|
||||
return;
|
||||
|
||||
release_resource(&host->mem_resource);
|
||||
host->mem_resource_registered = false;
|
||||
}
|
||||
|
||||
static int fake_pci_alloc_mem_resource(struct fake_pci_host *host,
|
||||
resource_size_t stride)
|
||||
{
|
||||
static const struct {
|
||||
resource_size_t start;
|
||||
resource_size_t end;
|
||||
const char *name;
|
||||
} regions[] = {
|
||||
{ FAKE_PCI_MEM_BASE, U32_MAX, "32-bit high" },
|
||||
{ 0xc0000000ULL, 0xcfffffffULL, "32-bit mid" },
|
||||
{ 0x80000000ULL, 0xbfffffffULL, "32-bit low" },
|
||||
#ifdef CONFIG_64BIT
|
||||
{ FAKE_PCI_MEM_64_BASE, (resource_size_t)-1, "64-bit" },
|
||||
#endif
|
||||
};
|
||||
int i, ret = -EBUSY;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(regions); i++) {
|
||||
host->mem_resource.start = 0;
|
||||
host->mem_resource.end = 0;
|
||||
host->mem_resource.flags = IORESOURCE_MEM;
|
||||
if (regions[i].start > U32_MAX)
|
||||
host->mem_resource.flags |= IORESOURCE_MEM_64;
|
||||
|
||||
ret = allocate_resource(&iomem_resource, &host->mem_resource,
|
||||
stride, regions[i].start, regions[i].end,
|
||||
stride, NULL, NULL);
|
||||
if (!ret) {
|
||||
host->mem_resource_registered = true;
|
||||
pr_info("fake_pci: allocated %s MMIO window [%#llx-%#llx]\n",
|
||||
regions[i].name,
|
||||
(unsigned long long)host->mem_resource.start,
|
||||
(unsigned long long)host->mem_resource.end);
|
||||
return 0;
|
||||
}
|
||||
|
||||
pr_warn("fake_pci: failed to allocate %s %#llx-byte MMIO window: %d\n",
|
||||
regions[i].name, (unsigned long long)stride, ret);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int fake_pci_host_init_mem_resource(struct fake_pci_host *host,
|
||||
unsigned int index)
|
||||
{
|
||||
resource_size_t max_addr = (resource_size_t)-1;
|
||||
resource_size_t mem_start;
|
||||
resource_size_t stride = mem_stride;
|
||||
int ret;
|
||||
|
||||
if (stride < FAKE_PCI_MEM_MIN_SIZE) {
|
||||
pr_err("fake_pci: mem_stride must be at least %#llx\n",
|
||||
(unsigned long long)FAKE_PCI_MEM_MIN_SIZE);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
host->mem_resource.flags = IORESOURCE_MEM;
|
||||
host->mem_resource.name = "fake_pci_mem";
|
||||
|
||||
if (mem_base) {
|
||||
mem_start = (resource_size_t)mem_base;
|
||||
if ((unsigned long)mem_start != mem_base ||
|
||||
index >= (max_addr - mem_start + 1) / stride) {
|
||||
pr_err("fake_pci: fixed MMIO window exceeds resource space\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
mem_start += index * stride;
|
||||
host->mem_resource.start = mem_start;
|
||||
host->mem_resource.end = mem_start + stride - 1;
|
||||
if (mem_start > U32_MAX)
|
||||
host->mem_resource.flags |= IORESOURCE_MEM_64;
|
||||
return 0;
|
||||
}
|
||||
|
||||
ret = fake_pci_alloc_mem_resource(host, stride);
|
||||
if (ret)
|
||||
pr_err("fake_pci: failed to allocate any MMIO window: %d\n", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int fake_pci_host_probe(struct fake_pci_host *host, unsigned int index)
|
||||
{
|
||||
struct pci_host_bridge *bridge;
|
||||
int err;
|
||||
|
||||
mutex_init(&host->lock);
|
||||
|
||||
host->domain_nr = index + 1;
|
||||
fake_pci_host_init_sysdata(host);
|
||||
|
||||
host->bus_resource.start = 0;
|
||||
host->bus_resource.end = 0;
|
||||
host->bus_resource.flags = IORESOURCE_BUS;
|
||||
host->bus_resource.name = "fake_pci_bus";
|
||||
|
||||
err = fake_pci_host_init_mem_resource(host, index);
|
||||
if (err)
|
||||
goto err_mutex;
|
||||
|
||||
init_pf_config_space(&host->pf);
|
||||
|
||||
bridge = pci_alloc_host_bridge(0);
|
||||
if (!bridge) {
|
||||
err = -ENOMEM;
|
||||
goto err_mem;
|
||||
}
|
||||
|
||||
host->bridge = bridge;
|
||||
bridge->sysdata = fake_pci_host_sysdata(host);
|
||||
bridge->ops = &fake_pci_ops;
|
||||
bridge->map_irq = fake_pci_map_irq;
|
||||
bridge->busnr = 0;
|
||||
bridge->dev.parent = &host->pdev->dev;
|
||||
bridge->domain_nr = host->domain_nr;
|
||||
|
||||
pci_set_host_bridge_release(bridge, fake_pci_release_host_bridge, NULL);
|
||||
|
||||
pci_add_resource(&bridge->windows, &host->bus_resource);
|
||||
pci_add_resource(&bridge->windows, &host->mem_resource);
|
||||
|
||||
mutex_lock(&fake_hosts_lock);
|
||||
list_add_tail(&host->list, &fake_hosts);
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
|
||||
err = pci_host_probe(bridge);
|
||||
if (err) {
|
||||
pr_err("fake_pci: pci_host_probe failed for domain %04x: %d\n",
|
||||
host->domain_nr, err);
|
||||
goto err_del_host;
|
||||
}
|
||||
|
||||
pr_info("fake_pci: Host bridge created on domain %04x\n",
|
||||
host->domain_nr);
|
||||
|
||||
return 0;
|
||||
|
||||
err_del_host:
|
||||
mutex_lock(&fake_hosts_lock);
|
||||
list_del(&host->list);
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
pci_free_host_bridge(bridge);
|
||||
err_mem:
|
||||
fake_pci_host_release_mem_resource(host);
|
||||
err_mutex:
|
||||
mutex_destroy(&host->lock);
|
||||
return err;
|
||||
}
|
||||
|
||||
static void fake_pci_host_remove(struct fake_pci_host *host)
|
||||
{
|
||||
struct platform_device *pdev;
|
||||
|
||||
if (!host)
|
||||
return;
|
||||
|
||||
if (host->bridge && host->bridge->bus) {
|
||||
pci_lock_rescan_remove();
|
||||
pci_stop_root_bus(host->bridge->bus);
|
||||
pci_remove_root_bus(host->bridge->bus);
|
||||
pci_unlock_rescan_remove();
|
||||
}
|
||||
|
||||
mutex_lock(&fake_hosts_lock);
|
||||
list_del_init(&host->list);
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
|
||||
fake_pci_host_release_mem_resource(host);
|
||||
|
||||
pdev = host->pdev;
|
||||
mutex_destroy(&host->lock);
|
||||
kfree(host);
|
||||
if (pdev)
|
||||
platform_device_unregister(pdev);
|
||||
}
|
||||
|
||||
static void fake_pci_remove_all_hosts(void)
|
||||
{
|
||||
struct fake_pci_host *host, *tmp;
|
||||
|
||||
list_for_each_entry_safe_reverse(host, tmp, &fake_hosts, list)
|
||||
fake_pci_host_remove(host);
|
||||
}
|
||||
|
||||
/* Forward declarations for drivers registered in init */
|
||||
extern struct pci_driver fake_pci_pf_driver;
|
||||
extern struct pci_driver pci_sim_vf_driver;
|
||||
extern struct pci_driver pci_sim_vfio_driver;
|
||||
|
||||
/* =========================================================================
|
||||
* Module init / exit
|
||||
* ========================================================================= */
|
||||
|
||||
static int __init fake_pci_sriov_init(void)
|
||||
{
|
||||
struct platform_device_info pdevinfo = {
|
||||
.name = "fake-pci-iommu",
|
||||
.id = PLATFORM_DEVID_AUTO,
|
||||
};
|
||||
struct fake_pci_host *host;
|
||||
unsigned int i;
|
||||
int err;
|
||||
|
||||
pr_info("fake_pci: Initializing fake PCI SR-IOV driver\n");
|
||||
|
||||
if (!num_pfs || num_pfs > FAKE_PCI_MAX_HOSTS) {
|
||||
pr_err("fake_pci: num_pfs must be between 1 and %u\n",
|
||||
FAKE_PCI_MAX_HOSTS);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
fake_iommu_pdev = platform_device_register_full(&pdevinfo);
|
||||
if (IS_ERR(fake_iommu_pdev)) {
|
||||
err = PTR_ERR(fake_iommu_pdev);
|
||||
pr_err("fake_pci: Failed to register IOMMU platform device: %d\n", err);
|
||||
return err;
|
||||
}
|
||||
|
||||
err = iommu_device_sysfs_add(&fake_iommu_dev, &fake_iommu_pdev->dev,
|
||||
NULL, "fake-pci-iommu");
|
||||
if (err) {
|
||||
pr_err("fake_pci: Failed to add IOMMU sysfs: %d\n", err);
|
||||
goto err_platform_dev;
|
||||
}
|
||||
|
||||
err = iommu_device_register(&fake_iommu_dev, &fake_iommu_ops,
|
||||
&fake_iommu_pdev->dev);
|
||||
if (err) {
|
||||
pr_err("fake_pci: Failed to register IOMMU: %d\n", err);
|
||||
goto err_iommu_sysfs;
|
||||
}
|
||||
|
||||
pr_info("fake_pci: Software IOMMU registered\n");
|
||||
|
||||
err = pci_sim_tty_register_driver();
|
||||
if (err) {
|
||||
pr_err("fake_pci: Failed to register TTY driver: %d\n", err);
|
||||
goto err_iommu;
|
||||
}
|
||||
|
||||
err = pci_register_driver(&pci_sim_vfio_driver);
|
||||
if (err) {
|
||||
pr_err("fake_pci: Failed to register VFIO UART driver: %d\n", err);
|
||||
goto err_tty;
|
||||
}
|
||||
|
||||
err = pci_register_driver(&pci_sim_vf_driver);
|
||||
if (err) {
|
||||
pr_err("fake_pci: Failed to register VF loopback driver: %d\n", err);
|
||||
goto err_vfio_driver;
|
||||
}
|
||||
|
||||
err = pci_register_driver(&fake_pci_pf_driver);
|
||||
if (err) {
|
||||
pr_err("fake_pci: Failed to register PF driver: %d\n", err);
|
||||
goto err_vf_driver;
|
||||
}
|
||||
|
||||
pdevinfo.name = "fake-pci-host";
|
||||
|
||||
for (i = 0; i < num_pfs; i++) {
|
||||
host = kzalloc(sizeof(*host), GFP_KERNEL);
|
||||
if (!host) {
|
||||
err = -ENOMEM;
|
||||
goto err_hosts;
|
||||
}
|
||||
INIT_LIST_HEAD(&host->list);
|
||||
|
||||
host->pdev = platform_device_register_full(&pdevinfo);
|
||||
if (IS_ERR(host->pdev)) {
|
||||
err = PTR_ERR(host->pdev);
|
||||
pr_err("fake_pci: Failed to register PCI host platform device %u: %d\n",
|
||||
i, err);
|
||||
kfree(host);
|
||||
goto err_hosts;
|
||||
}
|
||||
|
||||
err = fake_pci_host_probe(host, i);
|
||||
if (err) {
|
||||
platform_device_unregister(host->pdev);
|
||||
kfree(host);
|
||||
goto err_hosts;
|
||||
}
|
||||
}
|
||||
|
||||
pr_info("fake_pci: Module loaded successfully with %u PF(s)\n", num_pfs);
|
||||
pr_info("fake_pci: Use 'echo N > /sys/bus/pci/devices/.../sriov_numvfs' to enable VFs\n");
|
||||
|
||||
return 0;
|
||||
|
||||
err_hosts:
|
||||
fake_pci_remove_all_hosts();
|
||||
pci_unregister_driver(&fake_pci_pf_driver);
|
||||
err_vf_driver:
|
||||
pci_unregister_driver(&pci_sim_vf_driver);
|
||||
err_vfio_driver:
|
||||
pci_unregister_driver(&pci_sim_vfio_driver);
|
||||
err_tty:
|
||||
pci_sim_tty_unregister_driver();
|
||||
err_iommu:
|
||||
iommu_device_unregister(&fake_iommu_dev);
|
||||
err_iommu_sysfs:
|
||||
iommu_device_sysfs_remove(&fake_iommu_dev);
|
||||
err_platform_dev:
|
||||
platform_device_unregister(fake_iommu_pdev);
|
||||
return err;
|
||||
}
|
||||
|
||||
static void __exit fake_pci_sriov_exit(void)
|
||||
{
|
||||
pr_info("fake_pci: Unloading module\n");
|
||||
|
||||
fake_pci_remove_all_hosts();
|
||||
|
||||
pci_unregister_driver(&fake_pci_pf_driver);
|
||||
pci_unregister_driver(&pci_sim_vf_driver);
|
||||
pci_unregister_driver(&pci_sim_vfio_driver);
|
||||
pci_sim_tty_unregister_driver();
|
||||
|
||||
iommu_device_unregister(&fake_iommu_dev);
|
||||
iommu_device_sysfs_remove(&fake_iommu_dev);
|
||||
platform_device_unregister(fake_iommu_pdev);
|
||||
|
||||
pr_info("fake_pci: Module unloaded\n");
|
||||
}
|
||||
|
||||
module_init(fake_pci_sriov_init);
|
||||
module_exit(fake_pci_sriov_exit);
|
||||
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_AUTHOR("OpenStack Contributors");
|
||||
MODULE_DESCRIPTION("Fake PCI Host Controller with Software IOMMU for SR-IOV Testing");
|
||||
@@ -0,0 +1,242 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Fake PCI SR-IOV VFIO test fixture -- software IOMMU driver
|
||||
*
|
||||
* Implements a minimal IOMMU that claims all fake PCI domains and provides
|
||||
* software page-table tracking without any real DMA remapping. Every fake
|
||||
* PCI device gets its own IOMMU group so VFIO can assign individual VFs.
|
||||
*/
|
||||
|
||||
#include "fake_pci_sriov.h"
|
||||
|
||||
struct iommu_device fake_iommu_dev;
|
||||
struct platform_device *fake_iommu_pdev;
|
||||
|
||||
struct fake_iommu_domain {
|
||||
struct iommu_domain domain;
|
||||
struct xarray mappings;
|
||||
};
|
||||
|
||||
static struct fake_iommu_domain *to_fake_domain(struct iommu_domain *dom)
|
||||
{
|
||||
return container_of(dom, struct fake_iommu_domain, domain);
|
||||
}
|
||||
|
||||
static int fake_domain_attach(struct iommu_domain *domain, struct device *dev
|
||||
#if PCI_SIM_IOMMU_ATTACH_HAS_OLD_DOMAIN
|
||||
, struct iommu_domain *old
|
||||
#endif
|
||||
)
|
||||
{
|
||||
dev_dbg(dev, "fake_iommu: attached to domain type %u\n", domain->type);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct iommu_domain_ops fake_blocking_domain_ops = {
|
||||
.attach_dev = fake_domain_attach,
|
||||
};
|
||||
|
||||
static struct iommu_domain fake_blocking_domain = {
|
||||
.type = IOMMU_DOMAIN_BLOCKED,
|
||||
.ops = &fake_blocking_domain_ops,
|
||||
};
|
||||
|
||||
static void fake_domain_free_paging(struct iommu_domain *domain)
|
||||
{
|
||||
struct fake_iommu_domain *fake_dom = to_fake_domain(domain);
|
||||
|
||||
xa_destroy(&fake_dom->mappings);
|
||||
kfree(fake_dom);
|
||||
}
|
||||
|
||||
static int fake_domain_map_pages(struct iommu_domain *domain,
|
||||
unsigned long iova, phys_addr_t paddr,
|
||||
size_t pgsize, size_t pgcount, int prot,
|
||||
gfp_t gfp, size_t *mapped)
|
||||
{
|
||||
struct fake_iommu_domain *fake_dom = to_fake_domain(domain);
|
||||
unsigned long iova_pfn = iova >> PAGE_SHIFT;
|
||||
unsigned long pfn = paddr >> PAGE_SHIFT;
|
||||
size_t total = pgsize * pgcount;
|
||||
size_t npages = total >> PAGE_SHIFT;
|
||||
size_t i;
|
||||
int ret;
|
||||
|
||||
*mapped = 0;
|
||||
|
||||
if (!IS_ALIGNED(iova, PAGE_SIZE) || !IS_ALIGNED(paddr, PAGE_SIZE) ||
|
||||
!IS_ALIGNED(total, PAGE_SIZE))
|
||||
return -EINVAL;
|
||||
|
||||
for (i = 0; i < npages; i++) {
|
||||
if (xa_load(&fake_dom->mappings, iova_pfn + i)) {
|
||||
ret = -EBUSY;
|
||||
goto err_unmap;
|
||||
}
|
||||
|
||||
ret = xa_err(xa_store(&fake_dom->mappings, iova_pfn + i,
|
||||
xa_mk_value(pfn + i), gfp));
|
||||
if (ret)
|
||||
goto err_unmap;
|
||||
|
||||
*mapped += PAGE_SIZE;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
err_unmap:
|
||||
while (i--)
|
||||
xa_erase(&fake_dom->mappings, iova_pfn + i);
|
||||
*mapped = 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static size_t fake_domain_unmap_pages(struct iommu_domain *domain,
|
||||
unsigned long iova, size_t pgsize,
|
||||
size_t pgcount,
|
||||
struct iommu_iotlb_gather *gather)
|
||||
{
|
||||
struct fake_iommu_domain *fake_dom = to_fake_domain(domain);
|
||||
unsigned long iova_pfn = iova >> PAGE_SHIFT;
|
||||
size_t total = pgsize * pgcount;
|
||||
size_t npages = total >> PAGE_SHIFT;
|
||||
size_t unmapped = 0;
|
||||
|
||||
if (!IS_ALIGNED(iova, PAGE_SIZE) || !IS_ALIGNED(total, PAGE_SIZE))
|
||||
return 0;
|
||||
|
||||
while (unmapped < npages) {
|
||||
if (!xa_erase(&fake_dom->mappings, iova_pfn + unmapped))
|
||||
break;
|
||||
unmapped++;
|
||||
}
|
||||
|
||||
return unmapped << PAGE_SHIFT;
|
||||
}
|
||||
|
||||
static phys_addr_t fake_domain_iova_to_phys(struct iommu_domain *domain,
|
||||
dma_addr_t iova)
|
||||
{
|
||||
struct fake_iommu_domain *fake_dom = to_fake_domain(domain);
|
||||
void *entry;
|
||||
|
||||
entry = xa_load(&fake_dom->mappings, iova >> PAGE_SHIFT);
|
||||
if (!entry || !xa_is_value(entry))
|
||||
return 0;
|
||||
|
||||
return ((phys_addr_t)xa_to_value(entry) << PAGE_SHIFT) |
|
||||
(iova & ~PAGE_MASK);
|
||||
}
|
||||
|
||||
static const struct iommu_domain_ops fake_paging_domain_ops = {
|
||||
.attach_dev = fake_domain_attach,
|
||||
.free = fake_domain_free_paging,
|
||||
.map_pages = fake_domain_map_pages,
|
||||
.unmap_pages = fake_domain_unmap_pages,
|
||||
.iova_to_phys = fake_domain_iova_to_phys,
|
||||
};
|
||||
|
||||
static struct iommu_domain *fake_domain_alloc_paging_common(void)
|
||||
{
|
||||
struct fake_iommu_domain *fake_dom;
|
||||
|
||||
fake_dom = kzalloc(sizeof(*fake_dom), GFP_KERNEL);
|
||||
if (!fake_dom)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
fake_dom->domain.type = IOMMU_DOMAIN_UNMANAGED;
|
||||
fake_dom->domain.ops = &fake_paging_domain_ops;
|
||||
fake_dom->domain.pgsize_bitmap = PAGE_SIZE;
|
||||
fake_dom->domain.geometry.aperture_start = 0;
|
||||
fake_dom->domain.geometry.aperture_end = ~0UL;
|
||||
fake_dom->domain.geometry.force_aperture = true;
|
||||
xa_init(&fake_dom->mappings);
|
||||
|
||||
return &fake_dom->domain;
|
||||
}
|
||||
|
||||
#if PCI_SIM_IOMMU_HAS_DOMAIN_ALLOC_PAGING_FLAGS
|
||||
static struct iommu_domain *
|
||||
fake_domain_alloc_paging_flags(struct device *dev, u32 flags,
|
||||
const struct iommu_user_data *user_data)
|
||||
{
|
||||
if (flags & ~IOMMU_HWPT_ALLOC_PASID)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
if (user_data)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
return fake_domain_alloc_paging_common();
|
||||
}
|
||||
#else
|
||||
static struct iommu_domain *fake_domain_alloc_paging(struct device *dev)
|
||||
{
|
||||
return fake_domain_alloc_paging_common();
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool fake_iommu_capable(struct device *dev, enum iommu_cap cap)
|
||||
{
|
||||
switch (cap) {
|
||||
case IOMMU_CAP_CACHE_COHERENCY:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static struct iommu_device *fake_iommu_probe_device(struct device *dev)
|
||||
{
|
||||
struct fake_pci_host *host;
|
||||
struct pci_dev *pdev;
|
||||
bool found = false;
|
||||
int domain;
|
||||
|
||||
if (!dev_is_pci(dev))
|
||||
return ERR_PTR(-ENODEV);
|
||||
|
||||
pdev = to_pci_dev(dev);
|
||||
domain = pci_domain_nr(pdev->bus);
|
||||
|
||||
mutex_lock(&fake_hosts_lock);
|
||||
list_for_each_entry(host, &fake_hosts, list) {
|
||||
if (domain == host->domain_nr) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
mutex_unlock(&fake_hosts_lock);
|
||||
|
||||
if (!found)
|
||||
return ERR_PTR(-ENODEV);
|
||||
|
||||
dev_info(dev, "fake_iommu: probed device %04x:%02x:%02x.%d\n",
|
||||
pci_domain_nr(pdev->bus), pdev->bus->number,
|
||||
PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
|
||||
|
||||
return &fake_iommu_dev;
|
||||
}
|
||||
|
||||
static void fake_iommu_release_device(struct device *dev)
|
||||
{
|
||||
dev_dbg(dev, "fake_iommu: released device\n");
|
||||
}
|
||||
|
||||
static struct iommu_group *fake_iommu_device_group(struct device *dev)
|
||||
{
|
||||
return generic_device_group(dev);
|
||||
}
|
||||
|
||||
const struct iommu_ops fake_iommu_ops = {
|
||||
.owner = THIS_MODULE,
|
||||
.default_domain = &fake_blocking_domain,
|
||||
.blocked_domain = &fake_blocking_domain,
|
||||
.capable = fake_iommu_capable,
|
||||
.probe_device = fake_iommu_probe_device,
|
||||
.release_device = fake_iommu_release_device,
|
||||
.device_group = fake_iommu_device_group,
|
||||
#if PCI_SIM_IOMMU_HAS_DOMAIN_ALLOC_PAGING_FLAGS
|
||||
.domain_alloc_paging_flags = fake_domain_alloc_paging_flags,
|
||||
#else
|
||||
.domain_alloc_paging = fake_domain_alloc_paging,
|
||||
#endif
|
||||
};
|
||||
@@ -0,0 +1,593 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Fake PCI SR-IOV VFIO test fixture -- host-side VF TTY loopback driver
|
||||
*
|
||||
* Each fake VF is claimed by pci_sim_loopback_vf. The driver presents the VF
|
||||
* as a /dev/ttyPCI_SIM<N> character device backed by an in-kernel 16550-style
|
||||
* UART FIFO. Bytes written to the tty are echoed back so that a host-side
|
||||
* process (or the VFIO guest via the companion vfio.c driver) can verify the
|
||||
* loopback path end-to-end.
|
||||
*/
|
||||
|
||||
#include "fake_pci_sriov.h"
|
||||
|
||||
static DEFINE_IDR(pci_sim_tty_idr);
|
||||
static DEFINE_MUTEX(pci_sim_tty_idr_lock);
|
||||
static struct tty_driver *pci_sim_tty_driver;
|
||||
|
||||
/* =========================================================================
|
||||
* 16550-compatible UART emulation
|
||||
* ========================================================================= */
|
||||
|
||||
void pci_sim_uart_reset(struct pci_sim_uart *uart)
|
||||
{
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
memset(uart->regs, 0, sizeof(uart->regs));
|
||||
uart->head = 0;
|
||||
uart->tail = 0;
|
||||
uart->count = 0;
|
||||
uart->dlab = false;
|
||||
uart->overrun = false;
|
||||
uart->divisor = 0;
|
||||
uart->fcr = 0;
|
||||
uart->intr_trigger_level = 1;
|
||||
uart->regs[UART_LSR] = UART_LSR_TEMT | UART_LSR_THRE;
|
||||
uart->regs[UART_MSR] = UART_MSR_DSR | UART_MSR_DCD | UART_MSR_CTS;
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
}
|
||||
|
||||
void pci_sim_uart_init(struct pci_sim_uart *uart)
|
||||
{
|
||||
spin_lock_init(&uart->lock);
|
||||
pci_sim_uart_reset(uart);
|
||||
}
|
||||
|
||||
static unsigned int pci_sim_uart_space_locked(struct pci_sim_uart *uart)
|
||||
{
|
||||
return PCI_SIM_UART_FIFO_SIZE - uart->count;
|
||||
}
|
||||
|
||||
size_t pci_sim_uart_write_data(struct pci_sim_uart *uart,
|
||||
const u8 *buf, size_t len)
|
||||
{
|
||||
unsigned long flags;
|
||||
size_t copied, i;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
copied = min_t(size_t, len, pci_sim_uart_space_locked(uart));
|
||||
|
||||
for (i = 0; i < copied; i++) {
|
||||
uart->fifo[uart->head] = buf[i];
|
||||
uart->head = (uart->head + 1) % PCI_SIM_UART_FIFO_SIZE;
|
||||
uart->count++;
|
||||
}
|
||||
|
||||
uart->overrun = (copied < len);
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
|
||||
return copied;
|
||||
}
|
||||
|
||||
static size_t pci_sim_uart_peek_data(struct pci_sim_uart *uart, u8 *buf,
|
||||
size_t len)
|
||||
{
|
||||
unsigned long flags;
|
||||
size_t copied, i;
|
||||
unsigned int tail;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
copied = min_t(size_t, len, uart->count);
|
||||
tail = uart->tail;
|
||||
|
||||
for (i = 0; i < copied; i++) {
|
||||
buf[i] = uart->fifo[tail];
|
||||
tail = (tail + 1) % PCI_SIM_UART_FIFO_SIZE;
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
|
||||
return copied;
|
||||
}
|
||||
|
||||
static void pci_sim_uart_consume_data(struct pci_sim_uart *uart, size_t len)
|
||||
{
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
len = min_t(size_t, len, uart->count);
|
||||
uart->tail = (uart->tail + len) % PCI_SIM_UART_FIFO_SIZE;
|
||||
uart->count -= len;
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
}
|
||||
|
||||
unsigned int pci_sim_uart_write_room(struct pci_sim_uart *uart)
|
||||
{
|
||||
unsigned long flags;
|
||||
unsigned int room;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
room = pci_sim_uart_space_locked(uart);
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
|
||||
return room;
|
||||
}
|
||||
|
||||
unsigned int pci_sim_uart_chars_in_buffer(struct pci_sim_uart *uart)
|
||||
{
|
||||
unsigned long flags;
|
||||
unsigned int count;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
count = uart->count;
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
u8 pci_sim_uart_lsr(struct pci_sim_uart *uart)
|
||||
{
|
||||
unsigned long flags;
|
||||
u8 lsr = UART_LSR_TEMT | UART_LSR_THRE;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
if (uart->count)
|
||||
lsr |= UART_LSR_DR;
|
||||
if (uart->overrun)
|
||||
lsr |= UART_LSR_OE;
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
|
||||
return lsr;
|
||||
}
|
||||
|
||||
u8 pci_sim_uart_read_data(struct pci_sim_uart *uart)
|
||||
{
|
||||
unsigned long flags;
|
||||
u8 val = 0xff;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
if (uart->count) {
|
||||
val = uart->fifo[uart->tail];
|
||||
uart->tail = (uart->tail + 1) % PCI_SIM_UART_FIFO_SIZE;
|
||||
uart->count--;
|
||||
}
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
static u8 pci_sim_uart_iir(struct pci_sim_uart *uart)
|
||||
{
|
||||
unsigned long flags;
|
||||
u8 iir, ier;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
ier = uart->regs[UART_IER];
|
||||
|
||||
if ((ier & UART_IER_RLSI) && uart->overrun)
|
||||
iir = UART_IIR_RLSI;
|
||||
else if ((ier & UART_IER_RDI) &&
|
||||
uart->count >= uart->intr_trigger_level)
|
||||
iir = UART_IIR_RDI;
|
||||
else if (ier & UART_IER_THRI)
|
||||
iir = UART_IIR_THRI;
|
||||
else if ((ier & UART_IER_MSI) &&
|
||||
(uart->regs[UART_MCR] & (UART_MCR_RTS | UART_MCR_DTR)))
|
||||
iir = UART_IIR_MSI;
|
||||
else
|
||||
iir = UART_IIR_NO_INT;
|
||||
|
||||
if (uart->fcr & UART_FCR_ENABLE_FIFO)
|
||||
iir |= UART_IIR_FIFO_ENABLED_16550A;
|
||||
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
|
||||
return iir;
|
||||
}
|
||||
|
||||
static void pci_sim_uart_clear_fifo(struct pci_sim_uart *uart)
|
||||
{
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&uart->lock, flags);
|
||||
uart->head = 0;
|
||||
uart->tail = 0;
|
||||
uart->count = 0;
|
||||
uart->overrun = false;
|
||||
spin_unlock_irqrestore(&uart->lock, flags);
|
||||
}
|
||||
|
||||
u8 pci_sim_uart_read_reg(struct pci_sim_uart *uart, u8 reg)
|
||||
{
|
||||
reg &= 7;
|
||||
|
||||
if (uart->dlab) {
|
||||
switch (reg) {
|
||||
case UART_DLL:
|
||||
return uart->divisor & 0xff;
|
||||
case UART_DLM:
|
||||
return uart->divisor >> 8;
|
||||
}
|
||||
}
|
||||
|
||||
switch (reg) {
|
||||
case UART_RX:
|
||||
return pci_sim_uart_read_data(uart);
|
||||
case UART_IER:
|
||||
return uart->regs[UART_IER] & 0x0f;
|
||||
case UART_IIR:
|
||||
return pci_sim_uart_iir(uart);
|
||||
case UART_LCR:
|
||||
return uart->regs[UART_LCR];
|
||||
case UART_MCR:
|
||||
return uart->regs[UART_MCR];
|
||||
case UART_LSR:
|
||||
return pci_sim_uart_lsr(uart);
|
||||
case UART_MSR:
|
||||
return uart->regs[UART_MSR];
|
||||
case UART_SCR:
|
||||
return uart->regs[UART_SCR];
|
||||
default:
|
||||
return 0xff;
|
||||
}
|
||||
}
|
||||
|
||||
void pci_sim_uart_write_reg(struct pci_sim_uart *uart, u8 reg, u8 val)
|
||||
{
|
||||
reg &= 7;
|
||||
|
||||
if (uart->dlab) {
|
||||
switch (reg) {
|
||||
case UART_DLL:
|
||||
uart->divisor = (uart->divisor & 0xff00) | val;
|
||||
return;
|
||||
case UART_DLM:
|
||||
uart->divisor = (uart->divisor & 0x00ff) | (val << 8);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
switch (reg) {
|
||||
case UART_TX:
|
||||
pci_sim_uart_write_data(uart, &val, 1);
|
||||
break;
|
||||
case UART_IER:
|
||||
uart->regs[UART_IER] = val & 0x0f;
|
||||
break;
|
||||
case UART_FCR:
|
||||
uart->fcr = val;
|
||||
switch (val & UART_FCR_TRIGGER_MASK) {
|
||||
case UART_FCR_TRIGGER_1:
|
||||
uart->intr_trigger_level = 1;
|
||||
break;
|
||||
case UART_FCR_TRIGGER_4:
|
||||
uart->intr_trigger_level = 4;
|
||||
break;
|
||||
case UART_FCR_TRIGGER_8:
|
||||
uart->intr_trigger_level = 8;
|
||||
break;
|
||||
case UART_FCR_TRIGGER_14:
|
||||
uart->intr_trigger_level = 14;
|
||||
break;
|
||||
}
|
||||
if (val & (UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT))
|
||||
pci_sim_uart_clear_fifo(uart);
|
||||
break;
|
||||
case UART_LCR:
|
||||
uart->regs[UART_LCR] = val;
|
||||
uart->dlab = !!(val & UART_LCR_DLAB);
|
||||
break;
|
||||
case UART_MCR:
|
||||
uart->regs[UART_MCR] = val;
|
||||
break;
|
||||
case UART_SCR:
|
||||
uart->regs[UART_SCR] = val;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* VF TTY loopback helper
|
||||
* ========================================================================= */
|
||||
|
||||
static void pci_sim_vf_flush_to_tty(struct pci_sim_vf_tty *vf)
|
||||
{
|
||||
u8 buf[PCI_SIM_UART_CHUNK];
|
||||
size_t copied, pending;
|
||||
unsigned int room;
|
||||
bool pushed = false;
|
||||
|
||||
for (;;) {
|
||||
room = tty_buffer_space_avail(&vf->port);
|
||||
if (!room)
|
||||
break;
|
||||
|
||||
pending = pci_sim_uart_peek_data(&vf->uart, buf,
|
||||
min_t(size_t, sizeof(buf), room));
|
||||
if (!pending)
|
||||
break;
|
||||
|
||||
copied = tty_insert_flip_string(&vf->port, buf, pending);
|
||||
if (!copied)
|
||||
break;
|
||||
|
||||
pci_sim_uart_consume_data(&vf->uart, copied);
|
||||
pushed = true;
|
||||
|
||||
if (copied < pending)
|
||||
break;
|
||||
}
|
||||
|
||||
if (pushed)
|
||||
tty_flip_buffer_push(&vf->port);
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* TTY driver operations
|
||||
* ========================================================================= */
|
||||
|
||||
static int pci_sim_tty_install(struct tty_driver *driver,
|
||||
struct tty_struct *tty)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf;
|
||||
struct tty_port *port;
|
||||
int ret;
|
||||
|
||||
mutex_lock(&pci_sim_tty_idr_lock);
|
||||
vf = idr_find(&pci_sim_tty_idr, tty->index);
|
||||
if (vf)
|
||||
port = tty_port_get(&vf->port);
|
||||
else
|
||||
port = NULL;
|
||||
mutex_unlock(&pci_sim_tty_idr_lock);
|
||||
|
||||
if (!port)
|
||||
return -ENODEV;
|
||||
|
||||
tty->driver_data = vf;
|
||||
ret = tty_port_install(port, driver, tty);
|
||||
if (ret)
|
||||
tty_port_put(port);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void pci_sim_tty_cleanup(struct tty_struct *tty)
|
||||
{
|
||||
tty_port_put(tty->port);
|
||||
}
|
||||
|
||||
static int pci_sim_tty_open(struct tty_struct *tty, struct file *filp)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf = tty->driver_data;
|
||||
int ret;
|
||||
|
||||
mutex_lock(&vf->state_lock);
|
||||
ret = vf->dead ? -ENODEV : 0;
|
||||
mutex_unlock(&vf->state_lock);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
return tty_port_open(tty->port, tty, filp);
|
||||
}
|
||||
|
||||
static void pci_sim_tty_close(struct tty_struct *tty, struct file *filp)
|
||||
{
|
||||
tty_port_close(tty->port, tty, filp);
|
||||
}
|
||||
|
||||
static ssize_t pci_sim_tty_write(struct tty_struct *tty, const u8 *buf,
|
||||
size_t len)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf = tty->driver_data;
|
||||
size_t copied;
|
||||
|
||||
if (!len)
|
||||
return 0;
|
||||
|
||||
mutex_lock(&vf->state_lock);
|
||||
if (vf->dead) {
|
||||
mutex_unlock(&vf->state_lock);
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
copied = pci_sim_uart_write_data(&vf->uart, buf, len);
|
||||
pci_sim_vf_flush_to_tty(vf);
|
||||
mutex_unlock(&vf->state_lock);
|
||||
|
||||
return copied;
|
||||
}
|
||||
|
||||
static unsigned int pci_sim_tty_write_room(struct tty_struct *tty)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf = tty->driver_data;
|
||||
|
||||
if (!vf)
|
||||
return 0;
|
||||
|
||||
return pci_sim_uart_write_room(&vf->uart);
|
||||
}
|
||||
|
||||
static unsigned int pci_sim_tty_chars_in_buffer(struct tty_struct *tty)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf = tty->driver_data;
|
||||
|
||||
if (!vf)
|
||||
return 0;
|
||||
|
||||
return pci_sim_uart_chars_in_buffer(&vf->uart);
|
||||
}
|
||||
|
||||
static void pci_sim_tty_hangup(struct tty_struct *tty)
|
||||
{
|
||||
tty_port_hangup(tty->port);
|
||||
}
|
||||
|
||||
static const struct tty_operations pci_sim_tty_ops = {
|
||||
.install = pci_sim_tty_install,
|
||||
.open = pci_sim_tty_open,
|
||||
.close = pci_sim_tty_close,
|
||||
.write = pci_sim_tty_write,
|
||||
.write_room = pci_sim_tty_write_room,
|
||||
.chars_in_buffer = pci_sim_tty_chars_in_buffer,
|
||||
.hangup = pci_sim_tty_hangup,
|
||||
.cleanup = pci_sim_tty_cleanup,
|
||||
};
|
||||
|
||||
static void pci_sim_tty_destruct_port(struct tty_port *port)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf = container_of(port, struct pci_sim_vf_tty,
|
||||
port);
|
||||
|
||||
mutex_lock(&pci_sim_tty_idr_lock);
|
||||
if (vf->id >= 0)
|
||||
idr_remove(&pci_sim_tty_idr, vf->id);
|
||||
mutex_unlock(&pci_sim_tty_idr_lock);
|
||||
|
||||
kfree(vf);
|
||||
}
|
||||
|
||||
static const struct tty_port_operations pci_sim_tty_port_ops = {
|
||||
.destruct = pci_sim_tty_destruct_port,
|
||||
};
|
||||
|
||||
/* =========================================================================
|
||||
* VF PCI driver
|
||||
* ========================================================================= */
|
||||
|
||||
static int pci_sim_vf_probe(struct pci_dev *pdev,
|
||||
const struct pci_device_id *id)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf;
|
||||
struct device *tty_dev;
|
||||
int ret;
|
||||
|
||||
ret = pci_enable_device(pdev);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
vf = kzalloc(sizeof(*vf), GFP_KERNEL);
|
||||
if (!vf) {
|
||||
ret = -ENOMEM;
|
||||
goto err_disable;
|
||||
}
|
||||
|
||||
vf->pdev = pdev;
|
||||
vf->id = -1;
|
||||
mutex_init(&vf->state_lock);
|
||||
pci_sim_uart_init(&vf->uart);
|
||||
|
||||
mutex_lock(&pci_sim_tty_idr_lock);
|
||||
ret = idr_alloc(&pci_sim_tty_idr, vf, 0, PCI_SIM_MAX_TTYS, GFP_KERNEL);
|
||||
mutex_unlock(&pci_sim_tty_idr_lock);
|
||||
if (ret < 0)
|
||||
goto err_free;
|
||||
vf->id = ret;
|
||||
|
||||
tty_port_init(&vf->port);
|
||||
vf->port.ops = &pci_sim_tty_port_ops;
|
||||
|
||||
tty_dev = tty_port_register_device(&vf->port, pci_sim_tty_driver,
|
||||
vf->id, &pdev->dev);
|
||||
if (IS_ERR(tty_dev)) {
|
||||
ret = PTR_ERR(tty_dev);
|
||||
goto err_put_port;
|
||||
}
|
||||
|
||||
pci_set_drvdata(pdev, vf);
|
||||
pci_info(pdev, "registered /dev/%s%d, lsr=0x%02x\n",
|
||||
PCI_SIM_TTY_NAME, vf->id, pci_sim_uart_lsr(&vf->uart));
|
||||
|
||||
return 0;
|
||||
|
||||
err_put_port:
|
||||
tty_port_put(&vf->port);
|
||||
pci_disable_device(pdev);
|
||||
return ret;
|
||||
err_free:
|
||||
kfree(vf);
|
||||
err_disable:
|
||||
pci_disable_device(pdev);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void pci_sim_vf_remove(struct pci_dev *pdev)
|
||||
{
|
||||
struct pci_sim_vf_tty *vf = pci_get_drvdata(pdev);
|
||||
|
||||
if (!vf)
|
||||
return;
|
||||
|
||||
mutex_lock(&vf->state_lock);
|
||||
vf->dead = true;
|
||||
mutex_unlock(&vf->state_lock);
|
||||
|
||||
tty_port_tty_hangup(&vf->port, false);
|
||||
tty_unregister_device(pci_sim_tty_driver, vf->id);
|
||||
pci_set_drvdata(pdev, NULL);
|
||||
pci_disable_device(pdev);
|
||||
tty_port_put(&vf->port);
|
||||
}
|
||||
|
||||
static const struct pci_device_id pci_sim_vf_ids[] = {
|
||||
{ PCI_DEVICE(FAKE_PCI_VENDOR_ID, FAKE_PCI_VF_DEVICE_ID) },
|
||||
{ }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(pci, pci_sim_vf_ids);
|
||||
|
||||
struct pci_driver pci_sim_vf_driver = {
|
||||
.name = "pci_sim_loopback_vf",
|
||||
.id_table = pci_sim_vf_ids,
|
||||
.probe = pci_sim_vf_probe,
|
||||
.remove = pci_sim_vf_remove,
|
||||
};
|
||||
|
||||
/* =========================================================================
|
||||
* TTY driver registration (called from core init/exit)
|
||||
* ========================================================================= */
|
||||
|
||||
int pci_sim_tty_register_driver(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
pci_sim_tty_driver = tty_alloc_driver(PCI_SIM_MAX_TTYS,
|
||||
TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV);
|
||||
if (IS_ERR(pci_sim_tty_driver))
|
||||
return PTR_ERR(pci_sim_tty_driver);
|
||||
|
||||
pci_sim_tty_driver->driver_name = "pci_sim_loopback";
|
||||
pci_sim_tty_driver->name = PCI_SIM_TTY_NAME;
|
||||
pci_sim_tty_driver->major = 0;
|
||||
pci_sim_tty_driver->minor_start = 0;
|
||||
pci_sim_tty_driver->type = TTY_DRIVER_TYPE_SERIAL;
|
||||
pci_sim_tty_driver->subtype = SERIAL_TYPE_NORMAL;
|
||||
pci_sim_tty_driver->init_termios = tty_std_termios;
|
||||
pci_sim_tty_driver->init_termios.c_cflag = B9600 | CS8 | CREAD |
|
||||
HUPCL | CLOCAL;
|
||||
pci_sim_tty_driver->init_termios.c_lflag &= ~(ECHO | ICANON);
|
||||
pci_sim_tty_driver->init_termios.c_oflag &= ~(OPOST | ONLCR);
|
||||
|
||||
tty_set_operations(pci_sim_tty_driver, &pci_sim_tty_ops);
|
||||
|
||||
ret = tty_register_driver(pci_sim_tty_driver);
|
||||
if (ret) {
|
||||
tty_driver_kref_put(pci_sim_tty_driver);
|
||||
pci_sim_tty_driver = NULL;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void pci_sim_tty_unregister_driver(void)
|
||||
{
|
||||
if (!pci_sim_tty_driver)
|
||||
return;
|
||||
|
||||
tty_unregister_driver(pci_sim_tty_driver);
|
||||
tty_driver_kref_put(pci_sim_tty_driver);
|
||||
pci_sim_tty_driver = NULL;
|
||||
idr_destroy(&pci_sim_tty_idr);
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Fake PCI SR-IOV VFIO test fixture -- VFIO PCI device driver
|
||||
*
|
||||
* Generic vfio-pci expects BAR0 to be backed by real host MMIO. The fake VFs
|
||||
* only exist behind fake pci_ops, so pci_iomap()/ioread() cannot service guest
|
||||
* BAR accesses. This override-only driver still exposes the VF via VFIO, but
|
||||
* traps BAR0 accesses and emulates a tiny 16550 loopback UART. When
|
||||
* vfio_guest_8250_compat is set, the guest config-space vendor/device IDs are
|
||||
* overlaid with the SGI IOC3 serial identity so the guest 8250_pci driver
|
||||
* binds automatically.
|
||||
*/
|
||||
|
||||
#include "fake_pci_sriov.h"
|
||||
|
||||
static int pci_sim_vfio_open_device(struct vfio_device *core_vdev)
|
||||
{
|
||||
struct pci_sim_vfio_vf *sim = container_of(core_vdev,
|
||||
struct pci_sim_vfio_vf, core.vdev);
|
||||
int ret;
|
||||
|
||||
ret = vfio_pci_core_enable(&sim->core);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
pci_sim_uart_reset(&sim->uart);
|
||||
vfio_pci_core_finish_enable(&sim->core);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool pci_sim_vfio_bar0_uart_reg(loff_t pos, u8 *reg)
|
||||
{
|
||||
loff_t offset;
|
||||
|
||||
if (vfio_guest_8250_compat) {
|
||||
if (pos < PCI_SIM_SGI_IOC3_UART_OFFSET ||
|
||||
pos >= PCI_SIM_SGI_IOC3_UART_OFFSET + 8)
|
||||
return false;
|
||||
offset = pos - PCI_SIM_SGI_IOC3_UART_OFFSET;
|
||||
} else {
|
||||
if (pos >= 8)
|
||||
return false;
|
||||
offset = pos;
|
||||
}
|
||||
|
||||
*reg = offset;
|
||||
return true;
|
||||
}
|
||||
|
||||
static ssize_t pci_sim_vfio_bar0_rw(struct pci_sim_vfio_vf *sim,
|
||||
char __user *buf, size_t count,
|
||||
loff_t *ppos, bool iswrite)
|
||||
{
|
||||
loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
|
||||
size_t done;
|
||||
u8 reg, val;
|
||||
|
||||
if (pos >= PCI_SIM_VFIO_BAR0_SIZE)
|
||||
return -EINVAL;
|
||||
|
||||
count = min_t(size_t, count, PCI_SIM_VFIO_BAR0_SIZE - pos);
|
||||
|
||||
mutex_lock(&sim->lock);
|
||||
for (done = 0; done < count; done++) {
|
||||
if (iswrite) {
|
||||
if (copy_from_user(&val, buf + done, 1)) {
|
||||
mutex_unlock(&sim->lock);
|
||||
return done ?: -EFAULT;
|
||||
}
|
||||
if (!pci_sim_vfio_bar0_uart_reg(pos + done, ®))
|
||||
continue;
|
||||
|
||||
if (vfio_uart_trace)
|
||||
pr_info("fake_pci: vfio uart W off=0x%llx reg=%u val=0x%02x\n",
|
||||
(unsigned long long)(pos + done), reg, val);
|
||||
pci_sim_uart_write_reg(&sim->uart, reg, val);
|
||||
} else {
|
||||
if (pci_sim_vfio_bar0_uart_reg(pos + done, ®)) {
|
||||
val = pci_sim_uart_read_reg(&sim->uart, reg);
|
||||
if (vfio_uart_trace)
|
||||
pr_info("fake_pci: vfio uart R off=0x%llx reg=%u val=0x%02x\n",
|
||||
(unsigned long long)(pos + done), reg, val);
|
||||
} else {
|
||||
val = 0xff;
|
||||
}
|
||||
if (copy_to_user(buf + done, &val, 1)) {
|
||||
mutex_unlock(&sim->lock);
|
||||
return done ?: -EFAULT;
|
||||
}
|
||||
}
|
||||
}
|
||||
mutex_unlock(&sim->lock);
|
||||
|
||||
*ppos += done;
|
||||
return done;
|
||||
}
|
||||
|
||||
static int pci_sim_vfio_copy_config_value(char __user *buf,
|
||||
loff_t pos, size_t count,
|
||||
unsigned int reg, const void *val,
|
||||
size_t val_size)
|
||||
{
|
||||
loff_t copy_offset;
|
||||
size_t copy_count, register_offset;
|
||||
|
||||
if (!vfio_pci_core_range_intersect_range(pos, count, reg, val_size,
|
||||
©_offset, ©_count,
|
||||
®ister_offset))
|
||||
return 0;
|
||||
|
||||
if (copy_to_user(buf + copy_offset, val + register_offset, copy_count))
|
||||
return -EFAULT;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static ssize_t pci_sim_vfio_read_config(struct vfio_device *core_vdev,
|
||||
char __user *buf, size_t count,
|
||||
loff_t *ppos)
|
||||
{
|
||||
loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
|
||||
__le16 val16;
|
||||
__le32 val32;
|
||||
ssize_t ret;
|
||||
size_t done;
|
||||
|
||||
ret = vfio_pci_core_read(core_vdev, buf, count, ppos);
|
||||
if (ret <= 0 || !vfio_guest_8250_compat)
|
||||
return ret;
|
||||
done = ret;
|
||||
|
||||
val16 = cpu_to_le16(PCI_VENDOR_ID_SGI);
|
||||
if (pci_sim_vfio_copy_config_value(buf, pos, done, PCI_VENDOR_ID,
|
||||
&val16, sizeof(val16)))
|
||||
return -EFAULT;
|
||||
|
||||
val16 = cpu_to_le16(PCI_DEVICE_ID_SGI_IOC3);
|
||||
if (pci_sim_vfio_copy_config_value(buf, pos, done, PCI_DEVICE_ID,
|
||||
&val16, sizeof(val16)))
|
||||
return -EFAULT;
|
||||
|
||||
val32 = cpu_to_le32(FAKE_PCI_SERIAL_CLASS << 8);
|
||||
if (pci_sim_vfio_copy_config_value(buf, pos, done, PCI_CLASS_REVISION,
|
||||
&val32, sizeof(val32)))
|
||||
return -EFAULT;
|
||||
|
||||
val16 = cpu_to_le16(0xff00);
|
||||
if (pci_sim_vfio_copy_config_value(buf, pos, done,
|
||||
PCI_SUBSYSTEM_VENDOR_ID,
|
||||
&val16, sizeof(val16)))
|
||||
return -EFAULT;
|
||||
|
||||
val16 = cpu_to_le16(0);
|
||||
if (pci_sim_vfio_copy_config_value(buf, pos, done, PCI_SUBSYSTEM_ID,
|
||||
&val16, sizeof(val16)))
|
||||
return -EFAULT;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static ssize_t pci_sim_vfio_read(struct vfio_device *core_vdev,
|
||||
char __user *buf, size_t count, loff_t *ppos)
|
||||
{
|
||||
struct pci_sim_vfio_vf *sim = container_of(core_vdev,
|
||||
struct pci_sim_vfio_vf, core.vdev);
|
||||
unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
|
||||
|
||||
if (!count)
|
||||
return 0;
|
||||
|
||||
if (index == VFIO_PCI_CONFIG_REGION_INDEX)
|
||||
return pci_sim_vfio_read_config(core_vdev, buf, count, ppos);
|
||||
|
||||
if (index == VFIO_PCI_BAR0_REGION_INDEX)
|
||||
return pci_sim_vfio_bar0_rw(sim, buf, count, ppos, false);
|
||||
|
||||
return vfio_pci_core_read(core_vdev, buf, count, ppos);
|
||||
}
|
||||
|
||||
static ssize_t pci_sim_vfio_write(struct vfio_device *core_vdev,
|
||||
const char __user *buf, size_t count,
|
||||
loff_t *ppos)
|
||||
{
|
||||
struct pci_sim_vfio_vf *sim = container_of(core_vdev,
|
||||
struct pci_sim_vfio_vf, core.vdev);
|
||||
unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
|
||||
|
||||
if (!count)
|
||||
return 0;
|
||||
|
||||
if (index == VFIO_PCI_BAR0_REGION_INDEX)
|
||||
return pci_sim_vfio_bar0_rw(sim, (char __user *)buf, count,
|
||||
ppos, true);
|
||||
|
||||
return vfio_pci_core_write(core_vdev, buf, count, ppos);
|
||||
}
|
||||
|
||||
#if PCI_SIM_VFIO_HAS_GET_REGION_INFO_CAPS
|
||||
static int pci_sim_vfio_get_region_info(struct vfio_device *core_vdev,
|
||||
struct vfio_region_info *info,
|
||||
struct vfio_info_cap *caps)
|
||||
{
|
||||
if (info->index != VFIO_PCI_BAR0_REGION_INDEX)
|
||||
return vfio_pci_ioctl_get_region_info(core_vdev, info, caps);
|
||||
|
||||
info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
|
||||
info->size = vfio_guest_8250_compat ? PCI_SIM_VFIO_BAR0_SIZE : BAR0_SIZE;
|
||||
info->flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE;
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
static long pci_sim_vfio_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
|
||||
unsigned long arg)
|
||||
{
|
||||
struct vfio_region_info info;
|
||||
void __user *uarg = (void __user *)arg;
|
||||
|
||||
if (cmd != VFIO_DEVICE_GET_REGION_INFO)
|
||||
return vfio_pci_core_ioctl(core_vdev, cmd, arg);
|
||||
|
||||
if (copy_from_user(&info, uarg, sizeof(info)))
|
||||
return -EFAULT;
|
||||
|
||||
if (info.index != VFIO_PCI_BAR0_REGION_INDEX)
|
||||
return vfio_pci_core_ioctl(core_vdev, cmd, arg);
|
||||
|
||||
if (info.argsz < sizeof(info))
|
||||
return -EINVAL;
|
||||
|
||||
info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
|
||||
info.size = vfio_guest_8250_compat ? PCI_SIM_VFIO_BAR0_SIZE : BAR0_SIZE;
|
||||
info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE;
|
||||
info.cap_offset = 0;
|
||||
|
||||
if (copy_to_user(uarg, &info, sizeof(info)))
|
||||
return -EFAULT;
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int pci_sim_vfio_mmap(struct vfio_device *core_vdev,
|
||||
struct vm_area_struct *vma)
|
||||
{
|
||||
unsigned int index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
|
||||
|
||||
if (index == VFIO_PCI_BAR0_REGION_INDEX)
|
||||
return -EINVAL;
|
||||
|
||||
return vfio_pci_core_mmap(core_vdev, vma);
|
||||
}
|
||||
|
||||
static const struct vfio_device_ops pci_sim_vfio_ops = {
|
||||
.name = "pci_sim_vfio_pci",
|
||||
.init = vfio_pci_core_init_dev,
|
||||
.release = vfio_pci_core_release_dev,
|
||||
.open_device = pci_sim_vfio_open_device,
|
||||
.close_device = vfio_pci_core_close_device,
|
||||
#if PCI_SIM_VFIO_HAS_GET_REGION_INFO_CAPS
|
||||
.ioctl = vfio_pci_core_ioctl,
|
||||
.get_region_info_caps = pci_sim_vfio_get_region_info,
|
||||
#else
|
||||
.ioctl = pci_sim_vfio_ioctl,
|
||||
#endif
|
||||
.device_feature = vfio_pci_core_ioctl_feature,
|
||||
.read = pci_sim_vfio_read,
|
||||
.write = pci_sim_vfio_write,
|
||||
.mmap = pci_sim_vfio_mmap,
|
||||
.request = vfio_pci_core_request,
|
||||
.match = vfio_pci_core_match,
|
||||
#if PCI_SIM_VFIO_HAS_MATCH_TOKEN_UUID
|
||||
.match_token_uuid = vfio_pci_core_match_token_uuid,
|
||||
#endif
|
||||
.bind_iommufd = vfio_iommufd_physical_bind,
|
||||
.unbind_iommufd = vfio_iommufd_physical_unbind,
|
||||
.attach_ioas = vfio_iommufd_physical_attach_ioas,
|
||||
.detach_ioas = vfio_iommufd_physical_detach_ioas,
|
||||
#if PCI_SIM_VFIO_HAS_PASID_IOAS
|
||||
.pasid_attach_ioas = vfio_iommufd_physical_pasid_attach_ioas,
|
||||
.pasid_detach_ioas = vfio_iommufd_physical_pasid_detach_ioas,
|
||||
#endif
|
||||
};
|
||||
|
||||
static int pci_sim_vfio_probe(struct pci_dev *pdev,
|
||||
const struct pci_device_id *id)
|
||||
{
|
||||
struct pci_sim_vfio_vf *sim;
|
||||
int ret;
|
||||
|
||||
sim = vfio_alloc_device(pci_sim_vfio_vf, core.vdev, &pdev->dev,
|
||||
&pci_sim_vfio_ops);
|
||||
if (IS_ERR(sim))
|
||||
return PTR_ERR(sim);
|
||||
|
||||
mutex_init(&sim->lock);
|
||||
pci_sim_uart_init(&sim->uart);
|
||||
dev_set_drvdata(&pdev->dev, &sim->core);
|
||||
|
||||
ret = vfio_pci_core_register_device(&sim->core);
|
||||
if (ret)
|
||||
goto err_put;
|
||||
|
||||
pci_info(pdev, "registered VFIO UART BAR0 emulator\n");
|
||||
return 0;
|
||||
|
||||
err_put:
|
||||
vfio_put_device(&sim->core.vdev);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void pci_sim_vfio_remove(struct pci_dev *pdev)
|
||||
{
|
||||
struct vfio_pci_core_device *core = dev_get_drvdata(&pdev->dev);
|
||||
struct pci_sim_vfio_vf *sim;
|
||||
|
||||
if (!core)
|
||||
return;
|
||||
|
||||
sim = container_of(core, struct pci_sim_vfio_vf, core);
|
||||
vfio_pci_core_unregister_device(&sim->core);
|
||||
vfio_put_device(&sim->core.vdev);
|
||||
}
|
||||
|
||||
static const struct pci_device_id pci_sim_vfio_ids[] = {
|
||||
{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(FAKE_PCI_VENDOR_ID,
|
||||
FAKE_PCI_VF_DEVICE_ID) },
|
||||
{ }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(pci, pci_sim_vfio_ids);
|
||||
|
||||
struct pci_driver pci_sim_vfio_driver = {
|
||||
.name = "pci_sim_vfio_pci",
|
||||
.id_table = pci_sim_vfio_ids,
|
||||
.probe = pci_sim_vfio_probe,
|
||||
.remove = pci_sim_vfio_remove,
|
||||
.err_handler = &vfio_pci_core_err_handlers,
|
||||
.driver_managed_dma = true,
|
||||
};
|
||||
Executable
+195
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
"""Boot a CirrOS guest with one fake VF and run probe commands.
|
||||
|
||||
This helper is launched by ``run_cirros_vfio_guest_probe.sh`` after the
|
||||
shell wrapper has loaded ``fake_pci_sriov``, created one VF, and bound that
|
||||
VF to ``pci_sim_vfio_pci``. It owns the interactive QEMU console handling:
|
||||
wait for the CirrOS login prompt, log in, run guest-side PCI/TTY probes, and
|
||||
fail if the probe marker is not seen before the deadline.
|
||||
|
||||
Usage::
|
||||
|
||||
IMAGE=/tmp/cirros.img VF=0000:00:01.1 QEMU_DEADLINE=120 \
|
||||
python3 run_cirros_vfio_guest_probe.py
|
||||
|
||||
Environment variables:
|
||||
|
||||
* ``IMAGE``: required CirrOS qcow2 image path.
|
||||
* ``VF``: required PCI address of the VF bound to ``pci_sim_vfio_pci``.
|
||||
* ``QEMU_DEADLINE``: maximum seconds to wait for the guest probe; defaults to
|
||||
``120``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pty
|
||||
import re
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
GUEST_COMMANDS = (
|
||||
r"""
|
||||
echo GUEST_PROBE_BEGIN
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
echo DEV=$d vendor=$(cat $d/vendor) \
|
||||
device=$(cat $d/device) class=$(cat $d/class)
|
||||
cat $d/resource 2>/dev/null | head -1
|
||||
done
|
||||
ls -l /dev/ttyS* 2>/dev/null || true
|
||||
sudo dmesg | grep -Ei '1d55|1001|1d0f|8250|10a9|0003|serial|ttyS' || true
|
||||
for t in /dev/ttyS1 /dev/ttyS2 /dev/ttyS3 /dev/ttyS4; do
|
||||
[ -e $t ] && echo TRY_TTY=$t && \
|
||||
sudo stty -F $t raw -echo 9600 2>&1 && \
|
||||
sudo sh -c "echo -n hello > $t" 2>&1 && \
|
||||
timeout 2 sudo dd bs=1 count=5 if=$t 2>/dev/null | hexdump -C
|
||||
done
|
||||
echo GUEST_PROBE_END
|
||||
"""
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def launch_qemu(image: str, vf: str) -> tuple[subprocess.Popen[bytes], int]:
|
||||
"""Launch QEMU with ``vf`` assigned and return the process and PTY FD."""
|
||||
cmd = [
|
||||
"sudo",
|
||||
"-n",
|
||||
"qemu-system-x86_64",
|
||||
"-nodefaults",
|
||||
"-display",
|
||||
"none",
|
||||
"-serial",
|
||||
"stdio",
|
||||
"-monitor",
|
||||
"none",
|
||||
"-machine",
|
||||
"q35,accel=kvm",
|
||||
"-cpu",
|
||||
"host",
|
||||
"-smp",
|
||||
"1",
|
||||
"-m",
|
||||
"512M",
|
||||
"-snapshot",
|
||||
"-drive",
|
||||
f"file={image},if=virtio,format=qcow2",
|
||||
"-netdev",
|
||||
"user,id=n0",
|
||||
"-device",
|
||||
"virtio-net-pci,netdev=n0",
|
||||
"-device",
|
||||
f"vfio-pci,host={vf}",
|
||||
"-no-reboot",
|
||||
]
|
||||
print("+ " + " ".join(cmd), flush=True)
|
||||
master, slave = pty.openpty()
|
||||
proc = subprocess.Popen(
|
||||
cmd, stdin=slave, stdout=slave, stderr=slave, close_fds=True
|
||||
)
|
||||
os.close(slave)
|
||||
return proc, master
|
||||
|
||||
|
||||
def send(master: int, text: str) -> None:
|
||||
"""Send ``text`` to the QEMU serial console PTY."""
|
||||
os.write(master, text.encode())
|
||||
|
||||
|
||||
def read_console(master: int, timeout: float = 0.5) -> str:
|
||||
"""Read available QEMU console output for up to ``timeout`` seconds."""
|
||||
out = b""
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
rlist, _, _ = select.select(
|
||||
[master], [], [], max(0, end - time.monotonic())
|
||||
)
|
||||
if not rlist:
|
||||
break
|
||||
try:
|
||||
chunk = os.read(master, 4096)
|
||||
except OSError:
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
out += chunk
|
||||
if len(chunk) < 4096:
|
||||
break
|
||||
return out.decode(errors="replace")
|
||||
|
||||
|
||||
def run_guest_commands(
|
||||
proc: subprocess.Popen[bytes],
|
||||
master: int,
|
||||
deadline_s: int,
|
||||
) -> bool:
|
||||
"""Log in to CirrOS, run probe commands, and return success status."""
|
||||
buf = ""
|
||||
logged = False
|
||||
sent = False
|
||||
end = time.monotonic() + deadline_s
|
||||
|
||||
while time.monotonic() < end:
|
||||
output = read_console(master)
|
||||
if output:
|
||||
sys.stdout.write(output)
|
||||
sys.stdout.flush()
|
||||
buf += output
|
||||
|
||||
if not logged and re.search(r"(?:^|[\r\n])[^\r\n]* login:\s*$", buf):
|
||||
send(master, "cirros\n")
|
||||
buf = ""
|
||||
continue
|
||||
if not logged and re.search(r"(?:^|[\r\n])Password:\s*$", buf):
|
||||
send(master, "gocubsgo\n")
|
||||
logged = True
|
||||
buf = ""
|
||||
time.sleep(1)
|
||||
continue
|
||||
if logged and not sent and ("$ " in buf or "# " in buf):
|
||||
send(master, GUEST_COMMANDS)
|
||||
sent = True
|
||||
buf = ""
|
||||
continue
|
||||
if "GUEST_PROBE_END" in buf:
|
||||
return True
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
return False
|
||||
|
||||
|
||||
def stop_qemu(proc: subprocess.Popen[bytes]) -> None:
|
||||
"""Terminate QEMU, escalating to kill if it does not exit promptly."""
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the CirrOS guest probe and exit non-zero on failure."""
|
||||
image = os.environ["IMAGE"]
|
||||
vf = os.environ["VF"]
|
||||
deadline_s = int(os.environ.get("QEMU_DEADLINE", "120"))
|
||||
|
||||
proc, master = launch_qemu(image, vf)
|
||||
try:
|
||||
success = run_guest_commands(proc, master, deadline_s)
|
||||
finally:
|
||||
stop_qemu(proc)
|
||||
os.close(master)
|
||||
|
||||
print(f"\nQEMU_RC={proc.returncode}")
|
||||
if not success:
|
||||
sys.exit("FAIL: guest probe did not complete")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
#
|
||||
# Boot a CirrOS guest with one fake_pci_sriov VF assigned through VFIO and run
|
||||
# guest-side discovery commands from the serial console.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - qemu-system-x86_64, python3, curl, timeout, sudo without a password
|
||||
# - a kernel that can load fake_pci_sriov.ko and vfio-pci
|
||||
# - a CirrOS qcow2 image, downloaded automatically when IMAGE is missing
|
||||
#
|
||||
# Important environment variables:
|
||||
# - MODULE: fake_pci_sriov module path, default ./fake_pci_sriov.ko
|
||||
# - MODULE_ARGS: optional arguments passed to insmod
|
||||
# - RELOAD_MODULE: remove an existing fake_pci_sriov instance first, default 1
|
||||
# - IMAGE: CirrOS image path, default /tmp/cirros-0.6.3-x86_64-disk.img
|
||||
# - IMAGE_URL: URL used to download IMAGE when it is absent
|
||||
# - VENDOR/PF_DEVICE/VF_DEVICE: fake PCI IDs, defaults 0x1d55/0x1000/0x1001
|
||||
# - QEMU_DEADLINE: seconds to wait for the guest probe, default 120
|
||||
# - LOG: log file, default /tmp/cirros_vfio_guest_probe.log
|
||||
#
|
||||
# Flow: preflight, ensure image, reload/load module, create one VF, bind the VF
|
||||
# to pci_sim_vfio_pci, launch QEMU with a user-mode NIC so CirrOS reaches the
|
||||
# login prompt, log in to CirrOS, and run guest probes.
|
||||
# Exit status is zero only when the guest prints GUEST_PROBE_END.
|
||||
#
|
||||
# Example:
|
||||
# sudo -n true && bash pci-sim/run_cirros_vfio_guest_probe.sh
|
||||
set -euo pipefail
|
||||
|
||||
MODULE=${MODULE:-./fake_pci_sriov.ko}
|
||||
MODULE_ARGS=${MODULE_ARGS:-}
|
||||
RELOAD_MODULE=${RELOAD_MODULE:-1}
|
||||
IMAGE=${IMAGE:-/tmp/cirros-0.6.3-x86_64-disk.img}
|
||||
IMAGE_URL=${IMAGE_URL:-https://github.com/cirros-dev/cirros/releases/download/0.6.3/cirros-0.6.3-x86_64-disk.img}
|
||||
VENDOR=${VENDOR:-0x1d55}
|
||||
PF_DEVICE=${PF_DEVICE:-0x1000}
|
||||
VF_DEVICE=${VF_DEVICE:-0x1001}
|
||||
LOG=${LOG:-/tmp/cirros_vfio_guest_probe.log}
|
||||
QEMU_DEADLINE=${QEMU_DEADLINE:-120}
|
||||
|
||||
: > "$LOG"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
msg() { echo "== $* =="; }
|
||||
|
||||
find_dev() {
|
||||
local device=$1
|
||||
local d
|
||||
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
[ "$(cat "$d/vendor" 2>/dev/null || true)" = "$VENDOR" ] || continue
|
||||
[ "$(cat "$d/device" 2>/dev/null || true)" = "$device" ] || continue
|
||||
basename "$d"
|
||||
return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
PF=""
|
||||
VF=""
|
||||
cleanup() {
|
||||
set +e
|
||||
msg cleanup
|
||||
if [ -n "${VF:-}" ] && [ -e "/sys/bus/pci/devices/$VF" ]; then
|
||||
if [ -e "/sys/bus/pci/devices/$VF/driver/unbind" ]; then
|
||||
echo "$VF" | sudo -n tee "/sys/bus/pci/devices/$VF/driver/unbind" >/dev/null
|
||||
fi
|
||||
echo '' | sudo -n tee "/sys/bus/pci/devices/$VF/driver_override" >/dev/null
|
||||
fi
|
||||
if [ -n "${PF:-}" ] && [ -e "/sys/bus/pci/devices/$PF/sriov_numvfs" ]; then
|
||||
echo 0 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null
|
||||
fi
|
||||
msg "log saved to $LOG"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
msg preflight
|
||||
command -v qemu-system-x86_64
|
||||
command -v python3
|
||||
sudo -n true
|
||||
|
||||
msg "ensure CirrOS image"
|
||||
if [ ! -s "$IMAGE" ]; then
|
||||
timeout 180s curl -L --fail --connect-timeout 20 \
|
||||
-o "$IMAGE.tmp" "$IMAGE_URL"
|
||||
mv "$IMAGE.tmp" "$IMAGE"
|
||||
fi
|
||||
|
||||
msg "prepare VFIO and fake PCI module"
|
||||
sudo -n modprobe vfio-pci
|
||||
if grep -q '^fake_pci_sriov ' /proc/modules && [ "$RELOAD_MODULE" = 1 ]; then
|
||||
old_pf=$(find_dev "$PF_DEVICE" || true)
|
||||
if [ -n "$old_pf" ]; then
|
||||
echo 1 | sudo -n tee "/sys/bus/pci/devices/$old_pf/remove" >/dev/null || true
|
||||
fi
|
||||
sudo -n rmmod fake_pci_sriov || true
|
||||
fi
|
||||
if ! grep -q '^fake_pci_sriov ' /proc/modules; then
|
||||
msg "insmod $MODULE $MODULE_ARGS"
|
||||
# shellcheck disable=SC2086
|
||||
sudo -n insmod "$MODULE" $MODULE_ARGS
|
||||
fi
|
||||
|
||||
msg "create one VF"
|
||||
PF=$(find_dev "$PF_DEVICE")
|
||||
msg "PF=$PF"
|
||||
echo 0 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null || true
|
||||
echo 1 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null
|
||||
sleep 1
|
||||
VF=$(find_dev "$VF_DEVICE")
|
||||
msg "VF=$VF"
|
||||
|
||||
msg "bind VF to pci_sim_vfio_pci"
|
||||
if [ -e /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts ]; then
|
||||
echo Y | sudo -n tee \
|
||||
/sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts >/dev/null
|
||||
fi
|
||||
if [ -e "/sys/bus/pci/devices/$VF/driver/unbind" ]; then
|
||||
echo "$VF" | sudo -n tee "/sys/bus/pci/devices/$VF/driver/unbind" >/dev/null
|
||||
fi
|
||||
echo pci_sim_vfio_pci | sudo -n tee "/sys/bus/pci/devices/$VF/driver_override" >/dev/null
|
||||
echo "$VF" | sudo -n tee /sys/bus/pci/drivers_probe >/dev/null
|
||||
readlink -f "/sys/bus/pci/devices/$VF/driver"
|
||||
|
||||
msg "boot CirrOS and run guest probe"
|
||||
export IMAGE VF QEMU_DEADLINE
|
||||
python3 "$SCRIPT_DIR/run_cirros_vfio_guest_probe.py"
|
||||
msg PASS
|
||||
Executable
+151
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
"""Monitor a CirrOS config-drive guest UART end-to-end test.
|
||||
|
||||
This helper is launched by ``run_cirros_vfio_userdata_echo.sh`` after the
|
||||
shell wrapper has built a config-drive, loaded ``fake_pci_sriov``, created one
|
||||
VF, and bound that VF to ``pci_sim_vfio_pci``. The config-drive user-data
|
||||
script runs inside the guest and prints ``E2E_END`` when it finishes. This
|
||||
helper starts QEMU, mirrors serial output to stdout, and fails if the guest
|
||||
reports ``E2E_FAIL`` or the end marker is not seen before the deadline.
|
||||
|
||||
Usage::
|
||||
|
||||
IMAGE=/tmp/cirros.img VF=0000:00:01.1 CONFIG_ISO=/tmp/configdrive.iso \
|
||||
QEMU_DEADLINE=150 python3 run_cirros_vfio_userdata_echo.py
|
||||
|
||||
Environment variables:
|
||||
|
||||
* ``IMAGE``: required CirrOS qcow2 image path.
|
||||
* ``VF``: required PCI address of the VF bound to ``pci_sim_vfio_pci``.
|
||||
* ``CONFIG_ISO``: required config-drive ISO path containing the guest user-data
|
||||
script.
|
||||
* ``QEMU_DEADLINE``: maximum seconds to wait for ``E2E_END``; defaults to
|
||||
``150``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pty
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
MONITOR_INTERVAL = 0.5
|
||||
|
||||
|
||||
def launch_qemu(
|
||||
image: str,
|
||||
vf: str,
|
||||
config_iso: str,
|
||||
) -> tuple[subprocess.Popen[bytes], int]:
|
||||
"""Launch QEMU with ``vf`` assigned and ``config_iso`` attached."""
|
||||
cmd = [
|
||||
"sudo",
|
||||
"-n",
|
||||
"qemu-system-x86_64",
|
||||
"-nodefaults",
|
||||
"-display",
|
||||
"none",
|
||||
"-serial",
|
||||
"stdio",
|
||||
"-monitor",
|
||||
"none",
|
||||
"-machine",
|
||||
"q35,accel=kvm",
|
||||
"-cpu",
|
||||
"host",
|
||||
"-smp",
|
||||
"1",
|
||||
"-m",
|
||||
"512M",
|
||||
"-snapshot",
|
||||
"-drive",
|
||||
f"file={image},if=virtio,format=qcow2",
|
||||
"-drive",
|
||||
f"file={config_iso},if=virtio,media=cdrom,readonly=on,format=raw",
|
||||
"-netdev",
|
||||
"user,id=n0",
|
||||
"-device",
|
||||
"virtio-net-pci,netdev=n0",
|
||||
"-device",
|
||||
f"vfio-pci,host={vf}",
|
||||
"-no-reboot",
|
||||
]
|
||||
print("+ " + " ".join(cmd), flush=True)
|
||||
master, slave = pty.openpty()
|
||||
proc = subprocess.Popen(
|
||||
cmd, stdin=slave, stdout=slave, stderr=slave, close_fds=True
|
||||
)
|
||||
os.close(slave)
|
||||
return proc, master
|
||||
|
||||
|
||||
def monitor_e2e_output(
|
||||
proc: subprocess.Popen[bytes],
|
||||
master: int,
|
||||
deadline: int,
|
||||
) -> tuple[bool, str]:
|
||||
"""Mirror console output until ``E2E_END``, process exit, or deadline."""
|
||||
buf = ""
|
||||
success = False
|
||||
end = time.monotonic() + deadline
|
||||
|
||||
while time.monotonic() < end:
|
||||
rlist, _, _ = select.select([master], [], [], MONITOR_INTERVAL)
|
||||
if rlist:
|
||||
try:
|
||||
data = os.read(master, 4096)
|
||||
except OSError:
|
||||
break
|
||||
if not data:
|
||||
break
|
||||
text = data.decode(errors="replace")
|
||||
sys.stdout.write(text)
|
||||
sys.stdout.flush()
|
||||
buf += text
|
||||
if "E2E_END" in buf:
|
||||
success = True
|
||||
break
|
||||
if proc.poll() is not None and not rlist:
|
||||
break
|
||||
return success, buf
|
||||
|
||||
|
||||
def stop_qemu(proc: subprocess.Popen[bytes]) -> None:
|
||||
"""Terminate QEMU, escalating to kill if it does not exit promptly."""
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the guest user-data echo test and exit non-zero on failure."""
|
||||
image = os.environ["IMAGE"]
|
||||
vf = os.environ["VF"]
|
||||
config_iso = os.environ["CONFIG_ISO"]
|
||||
deadline = int(os.environ.get("QEMU_DEADLINE", "150"))
|
||||
|
||||
proc, master = launch_qemu(image, vf, config_iso)
|
||||
try:
|
||||
success, buf = monitor_e2e_output(proc, master, deadline)
|
||||
finally:
|
||||
stop_qemu(proc)
|
||||
os.close(master)
|
||||
|
||||
print(f"\nQEMU_RC={proc.returncode}")
|
||||
if not success:
|
||||
sys.exit("FAIL: E2E_END not seen")
|
||||
if "E2E_FAIL" in buf:
|
||||
sys.exit("FAIL: guest reported E2E_FAIL")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
#
|
||||
# Boot a CirrOS guest with config-drive user-data and one fake_pci_sriov VF
|
||||
# assigned through VFIO. The guest user-data script probes the assigned VF and
|
||||
# verifies the emulated UART loopback path from inside the guest.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - qemu-system-x86_64, python3, xorriso, curl, timeout, sudo without a password
|
||||
# - a kernel that can load fake_pci_sriov.ko and vfio-pci
|
||||
# - a CirrOS qcow2 image, downloaded automatically when IMAGE is missing
|
||||
#
|
||||
# Important environment variables:
|
||||
# - MODULE: fake_pci_sriov module path, default ./fake_pci_sriov.ko
|
||||
# - MODULE_ARGS: optional arguments passed to insmod
|
||||
# - RELOAD_MODULE: remove an existing fake_pci_sriov instance first, default 1
|
||||
# - IMAGE: CirrOS image path, default /tmp/cirros-0.6.3-x86_64-disk.img
|
||||
# - IMAGE_URL: URL used to download IMAGE when it is absent
|
||||
# - VENDOR/PF_DEVICE/VF_DEVICE: fake PCI IDs, defaults 0x1d55/0x1000/0x1001
|
||||
# - QEMU_DEADLINE: seconds to wait for guest completion, default 150
|
||||
# - WORK: temporary config-drive directory, default /tmp/cirros-vfio-e2e
|
||||
# - LOG: log file, default /tmp/cirros_vfio_userdata_echo.log
|
||||
# - USER_DATA: guest user-data script, default cirros_vfio_userdata_echo.sh
|
||||
#
|
||||
# Flow: preflight, create config-drive, ensure image, reload/load module, create
|
||||
# one VF, bind the VF to pci_sim_vfio_pci, launch QEMU, and monitor serial
|
||||
# output. Exit status is zero only when the guest prints E2E_END without
|
||||
# E2E_FAIL.
|
||||
#
|
||||
# Example:
|
||||
# sudo -n true && bash pci-sim/run_cirros_vfio_userdata_echo.sh
|
||||
set -euo pipefail
|
||||
|
||||
MODULE=${MODULE:-./fake_pci_sriov.ko}
|
||||
MODULE_ARGS=${MODULE_ARGS:-}
|
||||
RELOAD_MODULE=${RELOAD_MODULE:-1}
|
||||
IMAGE=${IMAGE:-/tmp/cirros-0.6.3-x86_64-disk.img}
|
||||
IMAGE_URL=${IMAGE_URL:-https://github.com/cirros-dev/cirros/releases/download/0.6.3/cirros-0.6.3-x86_64-disk.img}
|
||||
VENDOR=${VENDOR:-0x1d55}
|
||||
PF_DEVICE=${PF_DEVICE:-0x1000}
|
||||
VF_DEVICE=${VF_DEVICE:-0x1001}
|
||||
QEMU_DEADLINE=${QEMU_DEADLINE:-150}
|
||||
WORK=${WORK:-/tmp/cirros-vfio-e2e}
|
||||
LOG=${LOG:-/tmp/cirros_vfio_userdata_echo.log}
|
||||
USER_DATA=${USER_DATA:-}
|
||||
|
||||
: > "$LOG"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
msg() { echo "== $* =="; }
|
||||
|
||||
find_dev() {
|
||||
local device=$1
|
||||
local d
|
||||
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
[ "$(cat "$d/vendor" 2>/dev/null || true)" = "$VENDOR" ] || continue
|
||||
[ "$(cat "$d/device" 2>/dev/null || true)" = "$device" ] || continue
|
||||
basename "$d"
|
||||
return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
PF=""
|
||||
VF=""
|
||||
cleanup() {
|
||||
set +e
|
||||
msg cleanup
|
||||
if [ -n "${VF:-}" ] && [ -e "/sys/bus/pci/devices/$VF" ]; then
|
||||
if [ -e "/sys/bus/pci/devices/$VF/driver/unbind" ]; then
|
||||
echo "$VF" | sudo -n tee "/sys/bus/pci/devices/$VF/driver/unbind" >/dev/null
|
||||
fi
|
||||
echo '' | sudo -n tee "/sys/bus/pci/devices/$VF/driver_override" >/dev/null
|
||||
fi
|
||||
if [ -n "${PF:-}" ] && [ -e "/sys/bus/pci/devices/$PF/sriov_numvfs" ]; then
|
||||
echo 0 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null
|
||||
fi
|
||||
msg "log saved to $LOG"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
cd "$SCRIPT_DIR"
|
||||
USER_DATA=${USER_DATA:-$SCRIPT_DIR/cirros_vfio_userdata_echo.sh}
|
||||
|
||||
msg preflight
|
||||
command -v qemu-system-x86_64
|
||||
command -v python3
|
||||
command -v xorriso
|
||||
[ -r "$USER_DATA" ]
|
||||
sudo -n true
|
||||
|
||||
msg "ensure CirrOS image"
|
||||
if [ ! -s "$IMAGE" ]; then
|
||||
timeout 180s curl -L --fail --connect-timeout 20 \
|
||||
-o "$IMAGE.tmp" "$IMAGE_URL"
|
||||
mv "$IMAGE.tmp" "$IMAGE"
|
||||
fi
|
||||
|
||||
msg "create config-drive user_data"
|
||||
rm -rf "$WORK" && mkdir -p "$WORK/cfg/openstack/latest"
|
||||
cat > "$WORK/cfg/openstack/latest/meta_data.json" <<'JSON'
|
||||
{"uuid":"fake-pci-e2e-001","hostname":"cirros-fake-pci"}
|
||||
JSON
|
||||
cp "$USER_DATA" "$WORK/cfg/openstack/latest/user_data"
|
||||
chmod 755 "$WORK/cfg/openstack/latest/user_data"
|
||||
xorriso -as mkisofs -quiet -V config-2 -J -r \
|
||||
-o "$WORK/configdrive.iso" "$WORK/cfg"
|
||||
ls -lh "$WORK/configdrive.iso"
|
||||
|
||||
msg "prepare VFIO and fake PCI module"
|
||||
sudo -n modprobe vfio-pci
|
||||
if grep -q '^fake_pci_sriov ' /proc/modules && [ "$RELOAD_MODULE" = 1 ]; then
|
||||
msg "rmmod fake_pci_sriov"
|
||||
old_pf=$(find_dev "$PF_DEVICE" || true)
|
||||
if [ -n "$old_pf" ]; then
|
||||
echo 1 | sudo -n tee "/sys/bus/pci/devices/$old_pf/remove" >/dev/null || true
|
||||
fi
|
||||
sudo -n rmmod fake_pci_sriov || true
|
||||
fi
|
||||
if ! grep -q '^fake_pci_sriov ' /proc/modules; then
|
||||
msg "insmod $MODULE $MODULE_ARGS"
|
||||
# shellcheck disable=SC2086
|
||||
sudo -n insmod "$MODULE" $MODULE_ARGS
|
||||
fi
|
||||
|
||||
msg "create one VF"
|
||||
PF=$(find_dev "$PF_DEVICE")
|
||||
msg "PF=$PF"
|
||||
echo 0 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null || true
|
||||
echo 1 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null
|
||||
sleep 1
|
||||
VF=$(find_dev "$VF_DEVICE")
|
||||
msg "VF=$VF"
|
||||
|
||||
msg "bind VF to pci_sim_vfio_pci"
|
||||
if [ -e /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts ]; then
|
||||
echo Y | sudo -n tee \
|
||||
/sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts >/dev/null
|
||||
fi
|
||||
if [ -e "/sys/bus/pci/devices/$VF/driver/unbind" ]; then
|
||||
echo "$VF" | sudo -n tee "/sys/bus/pci/devices/$VF/driver/unbind" >/dev/null
|
||||
fi
|
||||
echo pci_sim_vfio_pci | sudo -n tee "/sys/bus/pci/devices/$VF/driver_override" >/dev/null
|
||||
echo "$VF" | sudo -n tee /sys/bus/pci/drivers_probe >/dev/null
|
||||
readlink -f "/sys/bus/pci/devices/$VF/driver"
|
||||
|
||||
msg "boot CirrOS with config-drive"
|
||||
export IMAGE VF QEMU_DEADLINE CONFIG_ISO="$WORK/configdrive.iso"
|
||||
python3 "$SCRIPT_DIR/run_cirros_vfio_userdata_echo.py"
|
||||
|
||||
msg "recent host dmesg"
|
||||
sudo -n dmesg | tail -80
|
||||
msg PASS
|
||||
Executable
+167
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
#
|
||||
# Host-side smoke test for fake_pci_sriov multiple-PF support.
|
||||
#
|
||||
# Purpose: load the module with multiple fake PFs, verify the expected PF/VF
|
||||
# count and class codes, ensure fake devices do not expose expansion ROMs, and
|
||||
# run cleanup at exit.
|
||||
#
|
||||
# Prerequisites: lspci, sudo without a password, vfio-pci, and a kernel that can
|
||||
# load fake_pci_sriov.ko.
|
||||
#
|
||||
# Important environment variables:
|
||||
# - MODULE: fake_pci_sriov module path, default ./fake_pci_sriov.ko
|
||||
# - MODULE_ARGS: module arguments, default num_pfs=2
|
||||
# - EXPECT_PFS: expected number of PFs, default 2
|
||||
# - EXPECT_PF_CLASS/EXPECT_VF_CLASS: expected class codes, default 0xff0000
|
||||
# - LOG: log file, default /tmp/fake_pci_multi_pf_smoke.log
|
||||
#
|
||||
# Exit status is zero when the expected topology appears and cleanup succeeds.
|
||||
# Example:
|
||||
# sudo -n true && bash pci-sim/run_fake_pci_multi_pf_smoke.sh
|
||||
set -euo pipefail
|
||||
|
||||
MODULE=${MODULE:-./fake_pci_sriov.ko}
|
||||
MODULE_ARGS=${MODULE_ARGS:-num_pfs=2}
|
||||
VENDOR=${VENDOR:-0x1d55}
|
||||
PF_DEVICE=${PF_DEVICE:-0x1000}
|
||||
VF_DEVICE=${VF_DEVICE:-0x1001}
|
||||
EXPECT_PFS=${EXPECT_PFS:-2}
|
||||
EXPECT_PF_CLASS=${EXPECT_PF_CLASS:-0xff0000}
|
||||
EXPECT_VF_CLASS=${EXPECT_VF_CLASS:-0xff0000}
|
||||
LOG=${LOG:-/tmp/fake_pci_multi_pf_smoke.log}
|
||||
|
||||
: > "$LOG"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
msg() { echo "== $* =="; }
|
||||
|
||||
find_fake_devs() {
|
||||
local device=$1 d
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
[ "$(cat "$d/vendor" 2>/dev/null || true)" = "$VENDOR" ] || continue
|
||||
[ "$(cat "$d/device" 2>/dev/null || true)" = "$device" ] || continue
|
||||
basename "$d"
|
||||
done | sort
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
set +e
|
||||
msg cleanup
|
||||
LOG=/tmp/fake_pci_multi_pf_cleanup.log \
|
||||
./cleanup_fake_pci_sriov.sh || true
|
||||
msg "log saved to $LOG"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
assert_class() {
|
||||
local dev=$1 expected=$2 class
|
||||
|
||||
class=$(cat "/sys/bus/pci/devices/$dev/class")
|
||||
echo "DEV=$dev CLASS=$class EXPECT_CLASS=$expected"
|
||||
[ "$class" = "$expected" ] || {
|
||||
echo "FAIL: $dev class $class, expected $expected"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
assert_no_rom() {
|
||||
local dev=$1
|
||||
|
||||
if lspci -D -s "$dev" -vv | grep -qi 'Expansion ROM'; then
|
||||
echo "FAIL: $dev unexpectedly has an expansion ROM resource"
|
||||
lspci -D -s "$dev" -vv
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
command -v lspci >/dev/null
|
||||
sudo -n true
|
||||
|
||||
if grep -q '^fake_pci_sriov ' /proc/modules ||
|
||||
[ -n "$(find_fake_devs "$PF_DEVICE")$(find_fake_devs "$VF_DEVICE")" ]; then
|
||||
LOG=/tmp/fake_pci_multi_pf_pre_cleanup.log \
|
||||
./cleanup_fake_pci_sriov.sh || true
|
||||
fi
|
||||
|
||||
msg "preload VFIO PCI dependencies"
|
||||
sudo -n modprobe vfio-pci
|
||||
|
||||
msg "insmod $MODULE $MODULE_ARGS"
|
||||
# shellcheck disable=SC2086
|
||||
sudo -n insmod "$MODULE" $MODULE_ARGS
|
||||
sleep 1
|
||||
|
||||
mapfile -t pfs < <(find_fake_devs "$PF_DEVICE")
|
||||
printf 'PF_LIST=%s\n' "${pfs[*]}"
|
||||
[ "${#pfs[@]}" -eq "$EXPECT_PFS" ] || {
|
||||
echo "FAIL: expected $EXPECT_PFS PFs, found ${#pfs[@]}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
for pf in "${pfs[@]}"; do
|
||||
assert_class "$pf" "$EXPECT_PF_CLASS"
|
||||
assert_no_rom "$pf"
|
||||
done
|
||||
|
||||
for pf in "${pfs[@]}"; do
|
||||
msg "enable VF on PF=$pf"
|
||||
echo 1 | sudo -n tee "/sys/bus/pci/devices/$pf/sriov_numvfs" >/dev/null
|
||||
done
|
||||
sleep 1
|
||||
|
||||
mapfile -t vfs < <(find_fake_devs "$VF_DEVICE")
|
||||
printf 'VF_LIST=%s\n' "${vfs[*]}"
|
||||
[ "${#vfs[@]}" -eq "$EXPECT_PFS" ] || {
|
||||
echo "FAIL: expected $EXPECT_PFS VFs, found ${#vfs[@]}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
for vf in "${vfs[@]}"; do
|
||||
assert_class "$vf" "$EXPECT_VF_CLASS"
|
||||
assert_no_rom "$vf"
|
||||
done
|
||||
|
||||
for dev in "${pfs[@]}" "${vfs[@]}"; do
|
||||
dev_path=/sys/bus/pci/devices/$dev
|
||||
driver=$(basename "$(readlink -f "$dev_path/driver" 2>/dev/null)" \
|
||||
2>/dev/null || true)
|
||||
group=$(basename "$(readlink -f "$dev_path/iommu_group" 2>/dev/null)" \
|
||||
2>/dev/null || true)
|
||||
echo "DEV=$dev DRIVER=$driver IOMMU_GROUP=$group"
|
||||
[ -n "$group" ] || {
|
||||
echo "FAIL: $dev has no IOMMU group"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
for pf in "${pfs[@]}"; do
|
||||
echo 0 | sudo -n tee "/sys/bus/pci/devices/$pf/sriov_numvfs" >/dev/null
|
||||
done
|
||||
sleep 1
|
||||
|
||||
mapfile -t vfs_after < <(find_fake_devs "$VF_DEVICE")
|
||||
printf 'VF_LIST_AFTER_DISABLE=%s\n' "${vfs_after[*]:-}"
|
||||
[ "${#vfs_after[@]}" -eq 0 ] || {
|
||||
echo "FAIL: VFs still visible after disable"
|
||||
exit 1
|
||||
}
|
||||
|
||||
msg "cleanup helper"
|
||||
LOG=/tmp/fake_pci_multi_pf_final_cleanup.log \
|
||||
./cleanup_fake_pci_sriov.sh
|
||||
|
||||
[ -z "$(find_fake_devs "$PF_DEVICE")$(find_fake_devs "$VF_DEVICE")" ] || {
|
||||
echo "FAIL: fake devices remain after cleanup"
|
||||
exit 1
|
||||
}
|
||||
! grep -q '^fake_pci_sriov ' /proc/modules || {
|
||||
echo "FAIL: fake_pci_sriov remains loaded after cleanup"
|
||||
exit 1
|
||||
}
|
||||
|
||||
trap - EXIT
|
||||
msg "log saved to $LOG"
|
||||
msg PASS
|
||||
Executable
+138
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
#
|
||||
# Bounded QEMU/VFIO smoke test for ./fake_pci_sriov.ko.
|
||||
#
|
||||
# Purpose: prove that a fake VF can be created, rebound from the host loopback
|
||||
# driver to pci_sim_vfio_pci, attached to QEMU with vfio-pci, and cleaned up.
|
||||
# This is a host-side smoke test; it only verifies QEMU can start with the VF.
|
||||
#
|
||||
# Prerequisites: qemu-system-x86_64, timeout, sudo without a password, vfio-pci,
|
||||
# and a kernel that can load fake_pci_sriov.ko.
|
||||
#
|
||||
# Important environment variables:
|
||||
# - MODULE: fake_pci_sriov module path, default ./fake_pci_sriov.ko
|
||||
# - MODULE_ARGS: optional arguments passed to insmod
|
||||
# - VFS: number of VFs to create on the first PF, default 1
|
||||
# - QEMU_TIMEOUT/QEMU_KILL_AFTER: bounds for the QEMU run, defaults 12s/5s
|
||||
# - ALLOW_UNSAFE_INTERRUPTS: set VFIO unsafe-interrupts knob when present,
|
||||
# default 1 for nested development environments
|
||||
# - LOG: log file, default /tmp/fake_pci_qemu_vfio_smoke.log
|
||||
#
|
||||
# Exit status is zero when QEMU starts and exits within the bounded timeout.
|
||||
# Example:
|
||||
# sudo -n true && bash pci-sim/run_fake_pci_qemu_vfio_smoke.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODULE=${MODULE:-./fake_pci_sriov.ko}
|
||||
MODULE_ARGS=${MODULE_ARGS:-}
|
||||
VENDOR=${VENDOR:-0x1d55}
|
||||
PF_DEVICE=${PF_DEVICE:-0x1000}
|
||||
VF_DEVICE=${VF_DEVICE:-0x1001}
|
||||
VFS=${VFS:-1}
|
||||
QEMU_TIMEOUT=${QEMU_TIMEOUT:-12s}
|
||||
QEMU_KILL_AFTER=${QEMU_KILL_AFTER:-5s}
|
||||
ALLOW_UNSAFE_INTERRUPTS=${ALLOW_UNSAFE_INTERRUPTS:-1}
|
||||
LOG=${LOG:-/tmp/fake_pci_qemu_vfio_smoke.log}
|
||||
|
||||
: > "$LOG"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
msg() { echo "== $* =="; }
|
||||
find_dev() {
|
||||
local device=$1
|
||||
local d
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
[ "$(cat "$d/vendor" 2>/dev/null || true)" = "$VENDOR" ] || continue
|
||||
[ "$(cat "$d/device" 2>/dev/null || true)" = "$device" ] || continue
|
||||
basename "$d"
|
||||
return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
PF=""
|
||||
VF=""
|
||||
cleanup() {
|
||||
set +e
|
||||
msg "cleanup"
|
||||
if [ -n "${VF:-}" ] && [ -e "/sys/bus/pci/devices/$VF" ]; then
|
||||
if [ -e "/sys/bus/pci/devices/$VF/driver/unbind" ]; then
|
||||
echo "$VF" | sudo -n tee "/sys/bus/pci/devices/$VF/driver/unbind" >/dev/null
|
||||
fi
|
||||
echo '' | sudo -n tee "/sys/bus/pci/devices/$VF/driver_override" >/dev/null
|
||||
fi
|
||||
if [ -n "${PF:-}" ] && [ -e "/sys/bus/pci/devices/$PF/sriov_numvfs" ]; then
|
||||
echo 0 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null
|
||||
fi
|
||||
msg "log saved to $LOG"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
msg "preflight"
|
||||
command -v qemu-system-x86_64
|
||||
command -v timeout
|
||||
sudo -n true
|
||||
|
||||
if ! grep -q '^fake_pci_sriov ' /proc/modules; then
|
||||
msg "loading fake_pci_sriov $MODULE_ARGS"
|
||||
# shellcheck disable=SC2086
|
||||
sudo -n insmod "$MODULE" $MODULE_ARGS
|
||||
else
|
||||
msg "fake_pci_sriov already loaded"
|
||||
fi
|
||||
|
||||
PF=$(find_dev "$PF_DEVICE")
|
||||
[ -n "$PF" ]
|
||||
msg "PF=$PF"
|
||||
|
||||
msg "enable $VFS VF"
|
||||
echo 0 | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null || true
|
||||
echo "$VFS" | sudo -n tee "/sys/bus/pci/devices/$PF/sriov_numvfs" >/dev/null
|
||||
sleep 1
|
||||
|
||||
VF=$(find_dev "$VF_DEVICE")
|
||||
[ -n "$VF" ]
|
||||
msg "VF=$VF"
|
||||
readlink -f "/sys/bus/pci/devices/$VF/driver" || true
|
||||
|
||||
msg "bind VF to pci_sim_vfio_pci"
|
||||
sudo -n modprobe vfio-pci
|
||||
if [ "$ALLOW_UNSAFE_INTERRUPTS" = 1 ] && \
|
||||
[ -e /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts ]; then
|
||||
msg "enable vfio_iommu_type1.allow_unsafe_interrupts for nested test VM"
|
||||
echo Y | sudo -n tee /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts >/dev/null
|
||||
fi
|
||||
if [ -e "/sys/bus/pci/devices/$VF/driver/unbind" ]; then
|
||||
echo "$VF" | sudo -n tee "/sys/bus/pci/devices/$VF/driver/unbind" >/dev/null
|
||||
fi
|
||||
echo pci_sim_vfio_pci | sudo -n tee "/sys/bus/pci/devices/$VF/driver_override" >/dev/null
|
||||
echo "$VF" | sudo -n tee /sys/bus/pci/drivers_probe >/dev/null
|
||||
readlink -f "/sys/bus/pci/devices/$VF/driver"
|
||||
|
||||
msg "QEMU/VFIO smoke test; timeout rc=124 is expected"
|
||||
set +e
|
||||
sudo -n timeout --foreground --kill-after="$QEMU_KILL_AFTER" "$QEMU_TIMEOUT" \
|
||||
qemu-system-x86_64 \
|
||||
-nodefaults -display none -serial none -parallel none -monitor none \
|
||||
-machine q35,accel=kvm \
|
||||
-m 256M \
|
||||
-device vfio-pci,host="$VF" \
|
||||
-S
|
||||
QEMU_RC=$?
|
||||
set -e
|
||||
echo "QEMU_RC=$QEMU_RC"
|
||||
|
||||
msg "recent kernel log"
|
||||
sudo -n dmesg | tail -160
|
||||
|
||||
if [ "$QEMU_RC" -eq 124 ] || [ "$QEMU_RC" -eq 0 ]; then
|
||||
msg "PASS"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
msg "FAIL: unexpected QEMU rc $QEMU_RC"
|
||||
exit "$QEMU_RC"
|
||||
Executable
+301
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
"""Basic host-side loopback test for ./fake_pci_sriov.ko.
|
||||
|
||||
The test finds the fake PF, enables VFs through sriov_numvfs, waits for the
|
||||
host-side tty devices, then verifies that bytes written to each tty are read
|
||||
back from the same tty.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import errno
|
||||
import os
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import termios
|
||||
import time
|
||||
import tty
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_NAME = "fake_pci_sriov"
|
||||
DEFAULT_MODULE = str(Path(__file__).resolve().with_name("fake_pci_sriov.ko"))
|
||||
|
||||
VENDOR = "0x1d55"
|
||||
PF_DEVICE = "0x1000"
|
||||
TTY_PREFIX = "/dev/ttyPCI_SIM"
|
||||
|
||||
|
||||
def run(cmd, check=True):
|
||||
print("+", " ".join(cmd), flush=True)
|
||||
return subprocess.run(cmd, check=check, text=True, capture_output=True)
|
||||
|
||||
|
||||
def module_loaded():
|
||||
try:
|
||||
with open("/proc/modules", encoding="utf-8") as fp:
|
||||
return any(line.split()[0] == MODULE_NAME for line in fp)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def module_refcnt():
|
||||
path = Path(f"/sys/module/{MODULE_NAME}/refcnt")
|
||||
if not path.exists():
|
||||
return None
|
||||
return int(path.read_text().strip())
|
||||
|
||||
|
||||
def read_text(path):
|
||||
return Path(path).read_text().strip().lower()
|
||||
|
||||
|
||||
def find_pf():
|
||||
for dev in sorted(Path("/sys/bus/pci/devices").iterdir()):
|
||||
try:
|
||||
if (
|
||||
read_text(dev / "vendor") == VENDOR
|
||||
and read_text(dev / "device") == PF_DEVICE
|
||||
):
|
||||
return dev
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def wait_for_pf(timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
pf = find_pf()
|
||||
if pf:
|
||||
return pf
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError("fake PCI PF 1d55:1000 did not appear")
|
||||
|
||||
|
||||
def wait_for_no_pf(timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if not find_pf():
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError("fake PCI PF did not disappear")
|
||||
|
||||
|
||||
def set_numvfs(pf, num_vfs):
|
||||
sriov_numvfs = pf / "sriov_numvfs"
|
||||
if sriov_numvfs.exists():
|
||||
sriov_numvfs.write_text(f"{num_vfs}\n")
|
||||
|
||||
|
||||
def remove_fake_pf(timeout):
|
||||
pf = find_pf()
|
||||
if not pf:
|
||||
return
|
||||
|
||||
try:
|
||||
set_numvfs(pf, 0)
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"warning: failed to disable VFs before remove: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
remove = pf / "remove"
|
||||
if remove.exists():
|
||||
remove.write_text("1\n")
|
||||
wait_for_no_pf(timeout)
|
||||
|
||||
|
||||
def unload_module(timeout, force_remove_pf=False):
|
||||
if not module_loaded():
|
||||
return
|
||||
|
||||
if force_remove_pf:
|
||||
remove_fake_pf(timeout)
|
||||
|
||||
refcnt = module_refcnt()
|
||||
if refcnt not in (None, 0):
|
||||
remove_fake_pf(timeout)
|
||||
|
||||
ret = run(["rmmod", MODULE_NAME], check=False)
|
||||
if ret.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"failed to unload {MODULE_NAME}: {ret.stderr.strip() or ret.stdout.strip()} "
|
||||
f"(refcnt={module_refcnt()})"
|
||||
)
|
||||
|
||||
|
||||
def load_module(module_path, timeout, reload_existing=False):
|
||||
if module_loaded():
|
||||
if not reload_existing:
|
||||
print(f"{MODULE_NAME} already loaded; reusing existing module")
|
||||
return
|
||||
unload_module(timeout, force_remove_pf=True)
|
||||
|
||||
# fake_pci_sriov uses vfio_pci_core symbols. insmod does not resolve
|
||||
# dependencies, so preload the VFIO PCI stack for standalone builds.
|
||||
run(["modprobe", "vfio-pci"])
|
||||
run(["insmod", module_path])
|
||||
wait_for_pf(timeout)
|
||||
|
||||
|
||||
def wait_for_ttys(count, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
paths = [Path(f"{TTY_PREFIX}{i}") for i in range(count)]
|
||||
while time.monotonic() < deadline:
|
||||
if all(p.exists() for p in paths):
|
||||
return paths
|
||||
time.sleep(0.1)
|
||||
missing = [str(p) for p in paths if not p.exists()]
|
||||
raise TimeoutError(f"TTY devices did not appear: {', '.join(missing)}")
|
||||
|
||||
|
||||
def set_raw(fd):
|
||||
attrs = termios.tcgetattr(fd)
|
||||
tty.setraw(fd)
|
||||
return attrs
|
||||
|
||||
|
||||
def restore_termios(fd, attrs):
|
||||
try:
|
||||
termios.tcsetattr(fd, termios.TCSANOW, attrs)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def read_exact(fd, size, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
data = bytearray()
|
||||
while len(data) < size:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
r, _, _ = select.select([fd], [], [], remaining)
|
||||
if not r:
|
||||
break
|
||||
try:
|
||||
chunk = os.read(fd, size - len(data))
|
||||
except BlockingIOError:
|
||||
continue
|
||||
if not chunk:
|
||||
break
|
||||
data.extend(chunk)
|
||||
return bytes(data)
|
||||
|
||||
|
||||
def test_tty(path, payload, timeout):
|
||||
fd = os.open(path, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
|
||||
old_attrs = None
|
||||
try:
|
||||
old_attrs = set_raw(fd)
|
||||
# Drain any stale data from a previous failed run.
|
||||
while True:
|
||||
try:
|
||||
if not os.read(fd, 4096):
|
||||
break
|
||||
except BlockingIOError:
|
||||
break
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
|
||||
break
|
||||
raise
|
||||
|
||||
written = os.write(fd, payload)
|
||||
if written != len(payload):
|
||||
raise RuntimeError(
|
||||
f"short write to {path}: {written}/{len(payload)}"
|
||||
)
|
||||
|
||||
got = read_exact(fd, len(payload), timeout)
|
||||
if got != payload:
|
||||
raise RuntimeError(
|
||||
f"loopback mismatch on {path}: got {got!r}, expected {payload!r}"
|
||||
)
|
||||
finally:
|
||||
if old_attrs is not None:
|
||||
restore_termios(fd, old_attrs)
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--module",
|
||||
default=DEFAULT_MODULE,
|
||||
help="module path for --load/--reload",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--load",
|
||||
"--insmod",
|
||||
dest="load",
|
||||
action="store_true",
|
||||
help="load the module before testing; reuse it if already loaded",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reload",
|
||||
action="store_true",
|
||||
help="deterministically remove any existing fake PF/module, then insmod",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--unload",
|
||||
"--rmmod",
|
||||
dest="unload",
|
||||
action="store_true",
|
||||
help="remove fake PF and rmmod the module after testing",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vfs", type=int, default=2, help="number of VFs to enable"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=5.0,
|
||||
help="wait/read timeout in seconds",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-vfs",
|
||||
action="store_true",
|
||||
help="leave VFs enabled after the test",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.reload:
|
||||
load_module(args.module, args.timeout, reload_existing=True)
|
||||
elif args.load:
|
||||
load_module(args.module, args.timeout, reload_existing=False)
|
||||
|
||||
pf = wait_for_pf(args.timeout)
|
||||
print(f"PF: {pf.name}")
|
||||
|
||||
try:
|
||||
# Start from a known VF state even when reusing an existing module.
|
||||
set_numvfs(pf, 0)
|
||||
pf = wait_for_pf(args.timeout)
|
||||
set_numvfs(pf, args.vfs)
|
||||
|
||||
ttys = wait_for_ttys(args.vfs, args.timeout)
|
||||
for i, path in enumerate(ttys):
|
||||
payload = f"pci-sim-vf{i}-hello\n".encode()
|
||||
test_tty(str(path), payload, args.timeout)
|
||||
print(f"ok: {path} echoed {payload!r}")
|
||||
finally:
|
||||
if args.unload:
|
||||
unload_module(args.timeout, force_remove_pf=True)
|
||||
elif not args.keep_vfs:
|
||||
try:
|
||||
pf = find_pf()
|
||||
if pf:
|
||||
set_numvfs(pf, 0)
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"warning: failed to disable VFs: {exc}", file=sys.stderr
|
||||
)
|
||||
|
||||
print("PASS")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+9
-2
@@ -62,8 +62,12 @@ intel_nic_driver = "cyborg.accelerator.drivers.nic.intel.driver:IntelNICDriver"
|
||||
inspur_nvme_ssd_driver = "cyborg.accelerator.drivers.ssd.inspur.driver:InspurNVMeSSDDriver"
|
||||
pci_driver = "cyborg.accelerator.drivers.pci.pci.driver:PCIDriver"
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["cyborg"]
|
||||
[tool.setuptools.packages.find]
|
||||
# pci-sim/ is an out-of-tree kernel module and must not be packaged in
|
||||
# cyborg wheels or sdists. Restricting auto-discovery to cyborg* ensures
|
||||
# it is excluded from both the wheel and the sdist (pbr.build delegates
|
||||
# to setuptools.build_meta for both).
|
||||
include = ["cyborg*"]
|
||||
|
||||
# NOTE: data-files is deprecated by setuptools in favour of package_data,
|
||||
# but package_data installs files inside the Python package tree and is not
|
||||
@@ -101,6 +105,9 @@ ignore = [
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"cyborg/tests/*" = ["S"]
|
||||
# pci-sim scripts are system-level test fixtures that intentionally call
|
||||
# subprocess with controlled arguments; suppress the subprocess audit rules.
|
||||
"pci-sim/*" = ["S"]
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
known-first-party = ["cyborg"]
|
||||
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
set -euo pipefail
|
||||
|
||||
KVER=${KVER:-$(uname -r)}
|
||||
CONFIG=${CONFIG:-/boot/config-$KVER}
|
||||
|
||||
if [[ ! -r "$CONFIG" ]]; then
|
||||
echo "ERROR: cannot read $CONFIG" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
require_y() {
|
||||
local opt=$1
|
||||
if ! grep -qx "${opt}=y" "$CONFIG"; then
|
||||
echo "ERROR: $opt must be built-in (=y) in $CONFIG" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_enabled() {
|
||||
local opt=$1
|
||||
if ! grep -Eq "^${opt}=(y|m)$" "$CONFIG"; then
|
||||
echo "ERROR: $opt must be enabled (=y or =m) in $CONFIG" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
warn_enabled() {
|
||||
local opt=$1
|
||||
if ! grep -Eq "^${opt}=(y|m)$" "$CONFIG"; then
|
||||
echo "WARNING: $opt is not enabled in $CONFIG" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
failed=0
|
||||
require_y CONFIG_MODULES || failed=1
|
||||
require_y CONFIG_PCI || failed=1
|
||||
require_y CONFIG_PCI_DOMAINS || failed=1
|
||||
require_y CONFIG_IOMMU_API || failed=1
|
||||
require_y CONFIG_TTY || failed=1
|
||||
require_enabled CONFIG_VFIO || failed=1
|
||||
require_enabled CONFIG_VFIO_PCI_CORE || failed=1
|
||||
|
||||
# Runtime/test dependencies for the supplied smoke tests.
|
||||
warn_enabled CONFIG_PCI_IOV
|
||||
warn_enabled CONFIG_VFIO_PCI
|
||||
warn_enabled CONFIG_KVM
|
||||
|
||||
if [[ $failed -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Kernel config dependencies look usable for pci-sim on $KVER"
|
||||
Executable
+365
@@ -0,0 +1,365 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Create pci-sim Nova and Cyborg test servers one at a time and validate the
|
||||
# VF-backed serial echo device inside CirrOS over SSH.
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
OS_CLOUD_NAME=${OS_CLOUD_NAME:-devstack-admin}
|
||||
IMAGE=${IMAGE:-cirros-0.6.3-x86_64-disk}
|
||||
NETWORK=${NETWORK:-private}
|
||||
PUBLIC_NETWORK=${PUBLIC_NETWORK:-public}
|
||||
NOVA_FLAVOR=${NOVA_FLAVOR:-pci-sim-nova}
|
||||
CYBORG_FLAVOR=${CYBORG_FLAVOR:-pci-sim-cyborg}
|
||||
NOVA_SERVER=${NOVA_SERVER:-pci-sim-test-nova-serial}
|
||||
CYBORG_SERVER=${CYBORG_SERVER:-pci-sim-test-cyborg-serial}
|
||||
SSH_USER=${SSH_USER:-cirros}
|
||||
SSH_PASSWORD=${SSH_PASSWORD:-gocubsgo}
|
||||
SSH_TIMEOUT=${SSH_TIMEOUT:-240}
|
||||
BUILD_TIMEOUT=${BUILD_TIMEOUT:-240}
|
||||
FLOATING_IP_TAG=${FLOATING_IP_TAG:-pci-sim-serial-echo-test}
|
||||
KEEP=${KEEP:-0}
|
||||
CLEANUP_ONLY=0
|
||||
RUN_NOVA=1
|
||||
RUN_CYBORG=1
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [options]
|
||||
|
||||
Create pci-sim test VMs serially, validate the passed-through VF-backed serial
|
||||
port from inside CirrOS, and clean up the VMs and floating IPs afterwards.
|
||||
|
||||
The script assumes DevStack has already created the image, flavors, device
|
||||
profile, Nova PCI config, and Cyborg PCI config.
|
||||
|
||||
By default this uses ``openstack --os-cloud devstack-admin`` and does not
|
||||
require sourcing ``openrc``:
|
||||
|
||||
$0
|
||||
|
||||
Defaults create one VM at a time on the private network and attach a temporary
|
||||
floating IP from the public network. The default security group for the current
|
||||
project is opened for ICMP and SSH.
|
||||
|
||||
Options:
|
||||
--os-cloud NAME clouds.yaml cloud name (default: $OS_CLOUD_NAME)
|
||||
--image NAME Image to boot (default: $IMAGE)
|
||||
--network NAME Tenant network for the VM (default: $NETWORK)
|
||||
--public-network NAME External network for floating IPs (default: $PUBLIC_NETWORK)
|
||||
--nova-flavor NAME Nova PCI flavor (default: $NOVA_FLAVOR)
|
||||
--cyborg-flavor NAME Cyborg flavor (default: $CYBORG_FLAVOR)
|
||||
--nova-server NAME Nova test server name (default: $NOVA_SERVER)
|
||||
--cyborg-server NAME Cyborg test server name (default: $CYBORG_SERVER)
|
||||
--ssh-user USER Guest SSH user (default: $SSH_USER)
|
||||
--ssh-password PASSWORD Guest SSH password (default: $SSH_PASSWORD)
|
||||
--build-timeout SECONDS Server ACTIVE timeout (default: $BUILD_TIMEOUT)
|
||||
--ssh-timeout SECONDS SSH readiness timeout (default: $SSH_TIMEOUT)
|
||||
--tag TAG Tag for temporary floating IPs (default: $FLOATING_IP_TAG)
|
||||
--nova-only Only run the Nova flavor test
|
||||
--cyborg-only Only run the Cyborg flavor test
|
||||
--keep Keep servers and floating IPs after validation
|
||||
--cleanup Delete test servers and floating IPs tagged by this script, then exit
|
||||
-h, --help Show this help
|
||||
|
||||
Environment variables with matching uppercase names may also be used for most
|
||||
options, e.g. OS_CLOUD_NAME=devstack-admin NETWORK=private PUBLIC_NETWORK=public $0.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--os-cloud) OS_CLOUD_NAME=$2; shift 2 ;;
|
||||
--image) IMAGE=$2; shift 2 ;;
|
||||
--network) NETWORK=$2; shift 2 ;;
|
||||
--public-network) PUBLIC_NETWORK=$2; shift 2 ;;
|
||||
--nova-flavor) NOVA_FLAVOR=$2; shift 2 ;;
|
||||
--cyborg-flavor) CYBORG_FLAVOR=$2; shift 2 ;;
|
||||
--nova-server) NOVA_SERVER=$2; shift 2 ;;
|
||||
--cyborg-server) CYBORG_SERVER=$2; shift 2 ;;
|
||||
--ssh-user) SSH_USER=$2; shift 2 ;;
|
||||
--ssh-password) SSH_PASSWORD=$2; shift 2 ;;
|
||||
--build-timeout) BUILD_TIMEOUT=$2; shift 2 ;;
|
||||
--ssh-timeout) SSH_TIMEOUT=$2; shift 2 ;;
|
||||
--tag) FLOATING_IP_TAG=$2; shift 2 ;;
|
||||
--nova-only) RUN_NOVA=1; RUN_CYBORG=0; shift ;;
|
||||
--cyborg-only) RUN_NOVA=0; RUN_CYBORG=1; shift ;;
|
||||
--keep) KEEP=1; shift ;;
|
||||
--cleanup) CLEANUP_ONLY=1; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || {
|
||||
echo "Required command not found: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
openstack() {
|
||||
command openstack --os-cloud "$OS_CLOUD_NAME" "$@"
|
||||
}
|
||||
|
||||
require_openstack_auth() {
|
||||
openstack token issue -f value -c id >/dev/null
|
||||
}
|
||||
|
||||
open_default_security_group() {
|
||||
local project_id sg_id
|
||||
|
||||
project_id=$(openstack token issue -f value -c project_id)
|
||||
sg_id=$(openstack security group list --project "$project_id" -f value -c ID -c Name |
|
||||
awk '$2 == "default" {print $1; exit}')
|
||||
if [[ -z $sg_id ]]; then
|
||||
echo "Could not find default security group for project $project_id" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Opening default security group $sg_id for ICMP and SSH"
|
||||
openstack security group rule create --ingress --protocol icmp "$sg_id" >/dev/null 2>&1 || true
|
||||
openstack security group rule create --ingress --protocol tcp --dst-port 22 "$sg_id" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
cleanup_resources() {
|
||||
local id name
|
||||
|
||||
echo "Cleaning up test servers"
|
||||
for name in "$NOVA_SERVER" "$CYBORG_SERVER"; do
|
||||
openstack server delete "$name" >/dev/null 2>&1 || true
|
||||
done
|
||||
for _ in $(seq 1 60); do
|
||||
if ! openstack server show "$NOVA_SERVER" >/dev/null 2>&1 && \
|
||||
! openstack server show "$CYBORG_SERVER" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "Deleting floating IPs tagged $FLOATING_IP_TAG"
|
||||
while read -r id; do
|
||||
[[ -n $id ]] || continue
|
||||
openstack floating ip delete "$id" >/dev/null 2>&1 || true
|
||||
done < <(openstack floating ip list --tags "$FLOATING_IP_TAG" -f value -c ID 2>/dev/null || true)
|
||||
}
|
||||
|
||||
write_guest_probe() {
|
||||
local path=$1
|
||||
cat > "$path" <<'GUEST'
|
||||
#!/bin/sh
|
||||
set -u
|
||||
PATH=/sbin:/bin:/usr/sbin:/usr/bin
|
||||
HOST=$(hostname 2>/dev/null || true)
|
||||
echo "=== GUEST $HOST uid=$(id 2>/dev/null) ==="
|
||||
|
||||
echo "PCI_MATCH_BEGIN"
|
||||
DEV=""
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
v=$(cat "$d/vendor" 2>/dev/null || true)
|
||||
p=$(cat "$d/device" 2>/dev/null || true)
|
||||
if { [ "$v" = "0x1d55" ] && [ "$p" = "0x1001" ]; } || \
|
||||
{ [ "$v" = "0x10a9" ] && [ "$p" = "0x0003" ]; } || \
|
||||
{ [ "$v" = "0x1d0f" ] && [ "$p" = "0x8250" ]; }; then
|
||||
DEV=$d
|
||||
echo "DEV=$DEV"
|
||||
echo "VENDOR=$v DEVICE=$p CLASS=$(cat "$d/class" 2>/dev/null || true)"
|
||||
echo "RESOURCE0=$(head -n1 "$d/resource" 2>/dev/null || true)"
|
||||
fi
|
||||
done
|
||||
[ -n "$DEV" ] || echo "NO_FAKE_VF"
|
||||
echo "PCI_MATCH_END"
|
||||
|
||||
echo "DMESG_MATCH_BEGIN"
|
||||
dmesg 2>/dev/null | grep -Ei '1d55|1001|10a9|0003|1d0f|8250|serial|ttyS' || true
|
||||
echo "DMESG_MATCH_END"
|
||||
|
||||
echo "SERIAL_ECHO_BEGIN"
|
||||
TEST_STR=ABCDEFGHIJKLMNOPQRSTUVWXYZ
|
||||
COUNT=$(printf "%s" "$TEST_STR" | wc -c)
|
||||
PASS=0
|
||||
for t in /dev/ttyS32 /dev/ttyS33 /dev/ttyS34 /dev/ttyS35 \
|
||||
/dev/ttyS1 /dev/ttyS2 /dev/ttyS3 /dev/ttyS4 /dev/ttyS5 \
|
||||
/dev/ttyS6 /dev/ttyS7 /dev/ttyS8 /dev/ttyS9; do
|
||||
[ -e "$t" ] || continue
|
||||
echo "TRY_TTY=$t"
|
||||
OUT=$(
|
||||
exec 7<>"$t" || exit 1
|
||||
stty -F "$t" raw -echo -icanon clocal -hupcl min 0 time 50 9600 2>&1 || exit 1
|
||||
printf "%s" "$TEST_STR" >&7
|
||||
TTY_READ=$(dd bs=1 count="$COUNT" <&7 2>/dev/null || true)
|
||||
echo "TTY_SELECTED=$t"
|
||||
echo "TTY_EXPECT=$TEST_STR"
|
||||
echo "TTY_READ=$TTY_READ"
|
||||
[ "$TTY_READ" = "$TEST_STR" ] || exit 2
|
||||
exit 0
|
||||
)
|
||||
rc=$?
|
||||
echo "$OUT"
|
||||
if [ $rc -eq 0 ]; then
|
||||
echo "SERIAL_ECHO_PASS=$t"
|
||||
PASS=1
|
||||
break
|
||||
else
|
||||
echo "TTY_RESULT=$t rc=$rc"
|
||||
fi
|
||||
done
|
||||
if [ "$PASS" = 1 ]; then
|
||||
echo "SERIAL_ECHO_OK"
|
||||
else
|
||||
echo "SERIAL_ECHO_FAIL"
|
||||
fi
|
||||
echo "SERIAL_ECHO_END"
|
||||
GUEST
|
||||
}
|
||||
|
||||
ssh_guest_probe() {
|
||||
local name=$1 ip=$2 guest_script=$3 log=$4
|
||||
|
||||
python3 - "$name" "$ip" "$SSH_USER" "$SSH_PASSWORD" "$guest_script" <<'PY' | tee "$log"
|
||||
import base64
|
||||
import os
|
||||
import pathlib
|
||||
import pexpect
|
||||
import sys
|
||||
|
||||
name, ip, user, password, guest_script = sys.argv[1:]
|
||||
payload = base64.b64encode(pathlib.Path(guest_script).read_bytes()).decode()
|
||||
cmd = (
|
||||
"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "
|
||||
"-o PreferredAuthentications=password -o PubkeyAuthentication=no "
|
||||
"-o ConnectTimeout=10 "
|
||||
f"{user}@{ip} "
|
||||
"'base64 -d >/tmp/pci-sim-serial-echo-test.sh; "
|
||||
f"echo {password} | sudo -S sh /tmp/pci-sim-serial-echo-test.sh'"
|
||||
)
|
||||
print(f"===== {name} {ip} guest validation =====", flush=True)
|
||||
child = pexpect.spawn(cmd, encoding="utf-8", timeout=60)
|
||||
child.logfile_read = sys.stdout
|
||||
try:
|
||||
while True:
|
||||
idx = child.expect([r"(?i)password:", r"(?i)yes/no", pexpect.EOF, pexpect.TIMEOUT])
|
||||
if idx == 0:
|
||||
child.sendline(password)
|
||||
break
|
||||
if idx == 1:
|
||||
child.sendline("yes")
|
||||
continue
|
||||
if idx == 2:
|
||||
raise SystemExit("SSH ended before password prompt")
|
||||
raise SystemExit("Timed out waiting for SSH password prompt")
|
||||
child.sendline(payload)
|
||||
child.sendeof()
|
||||
child.expect(pexpect.EOF, timeout=120)
|
||||
finally:
|
||||
child.close()
|
||||
if child.exitstatus != 0:
|
||||
raise SystemExit(child.exitstatus or 1)
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_active() {
|
||||
local name=$1 status
|
||||
local deadline=$((SECONDS + BUILD_TIMEOUT))
|
||||
|
||||
while (( SECONDS < deadline )); do
|
||||
status=$(openstack server show "$name" -f value -c status 2>/dev/null || echo MISSING)
|
||||
echo "$name status=$status"
|
||||
if [[ $status == ACTIVE ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ $status == ERROR ]]; then
|
||||
openstack server show "$name" -f yaml -c fault || true
|
||||
return 1
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
echo "$name did not become ACTIVE within ${BUILD_TIMEOUT}s" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_ssh() {
|
||||
local name=$1 ip=$2
|
||||
local deadline=$((SECONDS + SSH_TIMEOUT))
|
||||
|
||||
while (( SECONDS < deadline )); do
|
||||
if ping -c 1 -W 1 "$ip" >/dev/null 2>&1 && nc -z -w2 "$ip" 22; then
|
||||
echo "$name SSH ready at $ip"
|
||||
return 0
|
||||
fi
|
||||
echo "Waiting for SSH to $name at $ip"
|
||||
sleep 3
|
||||
done
|
||||
echo "$name did not become reachable by SSH within ${SSH_TIMEOUT}s" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_one() {
|
||||
local name=$1 flavor=$2
|
||||
local fip fip_id guest_script log
|
||||
|
||||
echo "===== Creating $name with flavor $flavor ====="
|
||||
openstack server delete "$name" >/dev/null 2>&1 || true
|
||||
for _ in $(seq 1 30); do
|
||||
openstack server show "$name" >/dev/null 2>&1 || break
|
||||
sleep 2
|
||||
done
|
||||
|
||||
openstack server create --image "$IMAGE" --flavor "$flavor" --network "$NETWORK" "$name"
|
||||
wait_for_active "$name"
|
||||
|
||||
fip=$(openstack floating ip create "$PUBLIC_NETWORK" \
|
||||
--tag "$FLOATING_IP_TAG" \
|
||||
--description "pci-sim serial echo test for $name" \
|
||||
-f value -c floating_ip_address)
|
||||
fip_id=$(openstack floating ip show "$fip" -f value -c id)
|
||||
echo "Associating floating IP $fip ($fip_id) to $name"
|
||||
openstack server add floating ip "$name" "$fip"
|
||||
|
||||
wait_for_ssh "$name" "$fip"
|
||||
|
||||
guest_script=$(mktemp)
|
||||
log=$(mktemp "/tmp/${name}.serial-echo.XXXXXX.log")
|
||||
write_guest_probe "$guest_script"
|
||||
ssh_guest_probe "$name" "$fip" "$guest_script" "$log"
|
||||
rm -f "$guest_script"
|
||||
|
||||
grep -q "SERIAL_ECHO_OK" "$log"
|
||||
grep -Eq "DEVICE=0x0003|DEVICE=0x1001|DEVICE=0x8250" "$log"
|
||||
echo "===== PASS $name (log: $log) ====="
|
||||
|
||||
if [[ $KEEP -eq 0 ]]; then
|
||||
echo "Cleaning up $name and floating IP $fip"
|
||||
openstack server delete "$name" >/dev/null 2>&1 || true
|
||||
for _ in $(seq 1 60); do
|
||||
openstack server show "$name" >/dev/null 2>&1 || break
|
||||
sleep 2
|
||||
done
|
||||
openstack floating ip delete "$fip_id" >/dev/null 2>&1 || true
|
||||
else
|
||||
echo "Keeping $name and floating IP $fip"
|
||||
fi
|
||||
}
|
||||
|
||||
require_command openstack
|
||||
require_command python3
|
||||
require_command ssh
|
||||
require_command nc
|
||||
require_command ping
|
||||
require_openstack_auth
|
||||
|
||||
if [[ $CLEANUP_ONLY -eq 1 ]]; then
|
||||
cleanup_resources
|
||||
exit 0
|
||||
fi
|
||||
|
||||
open_default_security_group
|
||||
|
||||
if [[ $RUN_NOVA -eq 1 ]]; then
|
||||
run_one "$NOVA_SERVER" "$NOVA_FLAVOR"
|
||||
fi
|
||||
if [[ $RUN_CYBORG -eq 1 ]]; then
|
||||
run_one "$CYBORG_SERVER" "$CYBORG_FLAVOR"
|
||||
fi
|
||||
|
||||
echo "All requested pci-sim serial echo tests passed."
|
||||
Reference in New Issue
Block a user