mirror of https://github.com/xemu-project/xemu.git
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1 iQEcBAABAgAGBQJiMCsfAAoJEO8Ells5jWIRtk0H/3xPsrgRCFJt62ITjD+0C86n eJBzsSkYqZIEMfQZ8Rp1QZTzfHf8T4OsqwA7kemTMvW44s4Z5Z+k1rktrIt0t0w0 ynAenxLqUGhHpSGfGQ2j1iy/DNRCX2MZfIO/Jo3SOzTpIVf4uGyJ/GR6i0iraXW0 +tOa/nutgnZFYrASmTBH2JgqQAieWrsW+tjxUK+yEC+eRbh2rrzbH8f91OeXxeH6 RGxmJPnW8XmEkIDIsiYotcrEYHUDYL+fiI5vf5whiEx2U1Vp/OWsBpzcVruOhgG3 tXI/d/gzRriH59Ly3rqGk/uAMrIIwzswobKp4kpfLiRdET4yJenhNIaBfHejS+w= =kiYq -----END PGP SIGNATURE----- Merge tag 'net-pull-request' of https://github.com/jasowang/qemu into staging # gpg: Signature made Tue 15 Mar 2022 05:58:55 GMT # gpg: using RSA key EF04965B398D6211 # gpg: Good signature from "Jason Wang (Jason Wang on RedHat) <jasowang@redhat.com>" [marginal] # gpg: WARNING: This key is not certified with sufficiently trusted signatures! # gpg: It is not certain that the signature belongs to the owner. # Primary key fingerprint: 215D 46F4 8246 689E C77F 3562 EF04 965B 398D 6211 * tag 'net-pull-request' of https://github.com/jasowang/qemu: vdpa: Expose VHOST_F_LOG_ALL on SVQ vdpa: Never set log_base addr if SVQ is enabled vdpa: Adapt vhost_vdpa_get_vring_base to SVQ vdpa: Add custom IOTLB translations to SVQ vhost: Add VhostIOVATree util: add iova_tree_find_iova util: Add iova_tree_alloc_map vhost: Shadow virtqueue buffers forwarding vdpa: adapt vhost_ops callbacks to svq virtio: Add vhost_svq_get_vring_addr vhost: Add vhost_svq_valid_features to shadow vq vhost: Add Shadow VirtQueue call forwarding capabilities vhost: Add Shadow VirtQueue kick forwarding capabilities vhost: Add VhostShadowVirtqueue virtio-net: fix map leaking on error during receive Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell
commit
a72ada1662
|
|
@ -1870,6 +1870,7 @@ static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf,
|
|||
|
||||
err:
|
||||
for (j = 0; j < i; j++) {
|
||||
virtqueue_detach_element(q->rx_vq, elems[j], lens[j]);
|
||||
g_free(elems[j]);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ softmmu_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-stub.c'))
|
|||
|
||||
virtio_ss = ss.source_set()
|
||||
virtio_ss.add(files('virtio.c'))
|
||||
virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c'))
|
||||
virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c', 'vhost-shadow-virtqueue.c', 'vhost-iova-tree.c'))
|
||||
virtio_ss.add(when: 'CONFIG_VHOST_USER', if_true: files('vhost-user.c'))
|
||||
virtio_ss.add(when: 'CONFIG_VHOST_VDPA', if_true: files('vhost-vdpa.c'))
|
||||
virtio_ss.add(when: 'CONFIG_VIRTIO_BALLOON', if_true: files('virtio-balloon.c'))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
/*
|
||||
* vhost software live migration iova tree
|
||||
*
|
||||
* SPDX-FileCopyrightText: Red Hat, Inc. 2021
|
||||
* SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#include "qemu/osdep.h"
|
||||
#include "qemu/iova-tree.h"
|
||||
#include "vhost-iova-tree.h"
|
||||
|
||||
#define iova_min_addr qemu_real_host_page_size
|
||||
|
||||
/**
|
||||
* VhostIOVATree, able to:
|
||||
* - Translate iova address
|
||||
* - Reverse translate iova address (from translated to iova)
|
||||
* - Allocate IOVA regions for translated range (linear operation)
|
||||
*/
|
||||
struct VhostIOVATree {
|
||||
/* First addressable iova address in the device */
|
||||
uint64_t iova_first;
|
||||
|
||||
/* Last addressable iova address in the device */
|
||||
uint64_t iova_last;
|
||||
|
||||
/* IOVA address to qemu memory maps. */
|
||||
IOVATree *iova_taddr_map;
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a new IOVA tree
|
||||
*
|
||||
* Returns the new IOVA tree
|
||||
*/
|
||||
VhostIOVATree *vhost_iova_tree_new(hwaddr iova_first, hwaddr iova_last)
|
||||
{
|
||||
VhostIOVATree *tree = g_new(VhostIOVATree, 1);
|
||||
|
||||
/* Some devices do not like 0 addresses */
|
||||
tree->iova_first = MAX(iova_first, iova_min_addr);
|
||||
tree->iova_last = iova_last;
|
||||
|
||||
tree->iova_taddr_map = iova_tree_new();
|
||||
return tree;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an iova tree
|
||||
*/
|
||||
void vhost_iova_tree_delete(VhostIOVATree *iova_tree)
|
||||
{
|
||||
iova_tree_destroy(iova_tree->iova_taddr_map);
|
||||
g_free(iova_tree);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the IOVA address stored from a memory address
|
||||
*
|
||||
* @tree: The iova tree
|
||||
* @map: The map with the memory address
|
||||
*
|
||||
* Return the stored mapping, or NULL if not found.
|
||||
*/
|
||||
const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree *tree,
|
||||
const DMAMap *map)
|
||||
{
|
||||
return iova_tree_find_iova(tree->iova_taddr_map, map);
|
||||
}
|
||||
|
||||
/**
|
||||
* Allocate a new mapping
|
||||
*
|
||||
* @tree: The iova tree
|
||||
* @map: The iova map
|
||||
*
|
||||
* Returns:
|
||||
* - IOVA_OK if the map fits in the container
|
||||
* - IOVA_ERR_INVALID if the map does not make sense (like size overflow)
|
||||
* - IOVA_ERR_NOMEM if tree cannot allocate more space.
|
||||
*
|
||||
* It returns assignated iova in map->iova if return value is VHOST_DMA_MAP_OK.
|
||||
*/
|
||||
int vhost_iova_tree_map_alloc(VhostIOVATree *tree, DMAMap *map)
|
||||
{
|
||||
/* Some vhost devices do not like addr 0. Skip first page */
|
||||
hwaddr iova_first = tree->iova_first ?: qemu_real_host_page_size;
|
||||
|
||||
if (map->translated_addr + map->size < map->translated_addr ||
|
||||
map->perm == IOMMU_NONE) {
|
||||
return IOVA_ERR_INVALID;
|
||||
}
|
||||
|
||||
/* Allocate a node in IOVA address */
|
||||
return iova_tree_alloc_map(tree->iova_taddr_map, map, iova_first,
|
||||
tree->iova_last);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove existing mappings from iova tree
|
||||
*
|
||||
* @iova_tree: The vhost iova tree
|
||||
* @map: The map to remove
|
||||
*/
|
||||
void vhost_iova_tree_remove(VhostIOVATree *iova_tree, const DMAMap *map)
|
||||
{
|
||||
iova_tree_remove(iova_tree->iova_taddr_map, map);
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* vhost software live migration iova tree
|
||||
*
|
||||
* SPDX-FileCopyrightText: Red Hat, Inc. 2021
|
||||
* SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef HW_VIRTIO_VHOST_IOVA_TREE_H
|
||||
#define HW_VIRTIO_VHOST_IOVA_TREE_H
|
||||
|
||||
#include "qemu/iova-tree.h"
|
||||
#include "exec/memory.h"
|
||||
|
||||
typedef struct VhostIOVATree VhostIOVATree;
|
||||
|
||||
VhostIOVATree *vhost_iova_tree_new(uint64_t iova_first, uint64_t iova_last);
|
||||
void vhost_iova_tree_delete(VhostIOVATree *iova_tree);
|
||||
G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostIOVATree, vhost_iova_tree_delete);
|
||||
|
||||
const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree *iova_tree,
|
||||
const DMAMap *map);
|
||||
int vhost_iova_tree_map_alloc(VhostIOVATree *iova_tree, DMAMap *map);
|
||||
void vhost_iova_tree_remove(VhostIOVATree *iova_tree, const DMAMap *map);
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,636 @@
|
|||
/*
|
||||
* vhost shadow virtqueue
|
||||
*
|
||||
* SPDX-FileCopyrightText: Red Hat, Inc. 2021
|
||||
* SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#include "qemu/osdep.h"
|
||||
#include "hw/virtio/vhost-shadow-virtqueue.h"
|
||||
|
||||
#include "qemu/error-report.h"
|
||||
#include "qapi/error.h"
|
||||
#include "qemu/main-loop.h"
|
||||
#include "qemu/log.h"
|
||||
#include "qemu/memalign.h"
|
||||
#include "linux-headers/linux/vhost.h"
|
||||
|
||||
/**
|
||||
* Validate the transport device features that both guests can use with the SVQ
|
||||
* and SVQs can use with the device.
|
||||
*
|
||||
* @dev_features: The features
|
||||
* @errp: Error pointer
|
||||
*/
|
||||
bool vhost_svq_valid_features(uint64_t features, Error **errp)
|
||||
{
|
||||
bool ok = true;
|
||||
uint64_t svq_features = features;
|
||||
|
||||
for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END;
|
||||
++b) {
|
||||
switch (b) {
|
||||
case VIRTIO_F_ANY_LAYOUT:
|
||||
continue;
|
||||
|
||||
case VIRTIO_F_ACCESS_PLATFORM:
|
||||
/* SVQ trust in the host's IOMMU to translate addresses */
|
||||
case VIRTIO_F_VERSION_1:
|
||||
/* SVQ trust that the guest vring is little endian */
|
||||
if (!(svq_features & BIT_ULL(b))) {
|
||||
svq_features |= BIT_ULL(b);
|
||||
ok = false;
|
||||
}
|
||||
continue;
|
||||
|
||||
default:
|
||||
if (svq_features & BIT_ULL(b)) {
|
||||
svq_features &= ~BIT_ULL(b);
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64
|
||||
", ok: 0x%"PRIx64, features, svq_features);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of descriptors that the SVQ can make available from the guest.
|
||||
*
|
||||
* @svq: The svq
|
||||
*/
|
||||
static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
|
||||
{
|
||||
return svq->vring.num - (svq->shadow_avail_idx - svq->shadow_used_idx);
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate addresses between the qemu's virtual address and the SVQ IOVA
|
||||
*
|
||||
* @svq: Shadow VirtQueue
|
||||
* @vaddr: Translated IOVA addresses
|
||||
* @iovec: Source qemu's VA addresses
|
||||
* @num: Length of iovec and minimum length of vaddr
|
||||
*/
|
||||
static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
|
||||
hwaddr *addrs, const struct iovec *iovec,
|
||||
size_t num)
|
||||
{
|
||||
if (num == 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < num; ++i) {
|
||||
DMAMap needle = {
|
||||
.translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
|
||||
.size = iovec[i].iov_len,
|
||||
};
|
||||
Int128 needle_last, map_last;
|
||||
size_t off;
|
||||
|
||||
const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
|
||||
/*
|
||||
* Map cannot be NULL since iova map contains all guest space and
|
||||
* qemu already has a physical address mapped
|
||||
*/
|
||||
if (unlikely(!map)) {
|
||||
qemu_log_mask(LOG_GUEST_ERROR,
|
||||
"Invalid address 0x%"HWADDR_PRIx" given by guest",
|
||||
needle.translated_addr);
|
||||
return false;
|
||||
}
|
||||
|
||||
off = needle.translated_addr - map->translated_addr;
|
||||
addrs[i] = map->iova + off;
|
||||
|
||||
needle_last = int128_add(int128_make64(needle.translated_addr),
|
||||
int128_make64(iovec[i].iov_len));
|
||||
map_last = int128_make64(map->translated_addr + map->size);
|
||||
if (unlikely(int128_gt(needle_last, map_last))) {
|
||||
qemu_log_mask(LOG_GUEST_ERROR,
|
||||
"Guest buffer expands over iova range");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void vhost_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
|
||||
const struct iovec *iovec, size_t num,
|
||||
bool more_descs, bool write)
|
||||
{
|
||||
uint16_t i = svq->free_head, last = svq->free_head;
|
||||
unsigned n;
|
||||
uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
|
||||
vring_desc_t *descs = svq->vring.desc;
|
||||
|
||||
if (num == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (n = 0; n < num; n++) {
|
||||
if (more_descs || (n + 1 < num)) {
|
||||
descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
|
||||
} else {
|
||||
descs[i].flags = flags;
|
||||
}
|
||||
descs[i].addr = cpu_to_le64(sg[n]);
|
||||
descs[i].len = cpu_to_le32(iovec[n].iov_len);
|
||||
|
||||
last = i;
|
||||
i = cpu_to_le16(descs[i].next);
|
||||
}
|
||||
|
||||
svq->free_head = le16_to_cpu(descs[last].next);
|
||||
}
|
||||
|
||||
static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
|
||||
VirtQueueElement *elem, unsigned *head)
|
||||
{
|
||||
unsigned avail_idx;
|
||||
vring_avail_t *avail = svq->vring.avail;
|
||||
bool ok;
|
||||
g_autofree hwaddr *sgs = g_new(hwaddr, MAX(elem->out_num, elem->in_num));
|
||||
|
||||
*head = svq->free_head;
|
||||
|
||||
/* We need some descriptors here */
|
||||
if (unlikely(!elem->out_num && !elem->in_num)) {
|
||||
qemu_log_mask(LOG_GUEST_ERROR,
|
||||
"Guest provided element with no descriptors");
|
||||
return false;
|
||||
}
|
||||
|
||||
ok = vhost_svq_translate_addr(svq, sgs, elem->out_sg, elem->out_num);
|
||||
if (unlikely(!ok)) {
|
||||
return false;
|
||||
}
|
||||
vhost_vring_write_descs(svq, sgs, elem->out_sg, elem->out_num,
|
||||
elem->in_num > 0, false);
|
||||
|
||||
|
||||
ok = vhost_svq_translate_addr(svq, sgs, elem->in_sg, elem->in_num);
|
||||
if (unlikely(!ok)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
vhost_vring_write_descs(svq, sgs, elem->in_sg, elem->in_num, false, true);
|
||||
|
||||
/*
|
||||
* Put the entry in the available array (but don't update avail->idx until
|
||||
* they do sync).
|
||||
*/
|
||||
avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
|
||||
avail->ring[avail_idx] = cpu_to_le16(*head);
|
||||
svq->shadow_avail_idx++;
|
||||
|
||||
/* Update the avail index after write the descriptor */
|
||||
smp_wmb();
|
||||
avail->idx = cpu_to_le16(svq->shadow_avail_idx);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool vhost_svq_add(VhostShadowVirtqueue *svq, VirtQueueElement *elem)
|
||||
{
|
||||
unsigned qemu_head;
|
||||
bool ok = vhost_svq_add_split(svq, elem, &qemu_head);
|
||||
if (unlikely(!ok)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
svq->ring_id_maps[qemu_head] = elem;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void vhost_svq_kick(VhostShadowVirtqueue *svq)
|
||||
{
|
||||
/*
|
||||
* We need to expose the available array entries before checking the used
|
||||
* flags
|
||||
*/
|
||||
smp_mb();
|
||||
if (svq->vring.used->flags & VRING_USED_F_NO_NOTIFY) {
|
||||
return;
|
||||
}
|
||||
|
||||
event_notifier_set(&svq->hdev_kick);
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward available buffers.
|
||||
*
|
||||
* @svq: Shadow VirtQueue
|
||||
*
|
||||
* Note that this function does not guarantee that all guest's available
|
||||
* buffers are available to the device in SVQ avail ring. The guest may have
|
||||
* exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in
|
||||
* qemu vaddr.
|
||||
*
|
||||
* If that happens, guest's kick notifications will be disabled until the
|
||||
* device uses some buffers.
|
||||
*/
|
||||
static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
|
||||
{
|
||||
/* Clear event notifier */
|
||||
event_notifier_test_and_clear(&svq->svq_kick);
|
||||
|
||||
/* Forward to the device as many available buffers as possible */
|
||||
do {
|
||||
virtio_queue_set_notification(svq->vq, false);
|
||||
|
||||
while (true) {
|
||||
VirtQueueElement *elem;
|
||||
bool ok;
|
||||
|
||||
if (svq->next_guest_avail_elem) {
|
||||
elem = g_steal_pointer(&svq->next_guest_avail_elem);
|
||||
} else {
|
||||
elem = virtqueue_pop(svq->vq, sizeof(*elem));
|
||||
}
|
||||
|
||||
if (!elem) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (elem->out_num + elem->in_num > vhost_svq_available_slots(svq)) {
|
||||
/*
|
||||
* This condition is possible since a contiguous buffer in GPA
|
||||
* does not imply a contiguous buffer in qemu's VA
|
||||
* scatter-gather segments. If that happens, the buffer exposed
|
||||
* to the device needs to be a chain of descriptors at this
|
||||
* moment.
|
||||
*
|
||||
* SVQ cannot hold more available buffers if we are here:
|
||||
* queue the current guest descriptor and ignore further kicks
|
||||
* until some elements are used.
|
||||
*/
|
||||
svq->next_guest_avail_elem = elem;
|
||||
return;
|
||||
}
|
||||
|
||||
ok = vhost_svq_add(svq, elem);
|
||||
if (unlikely(!ok)) {
|
||||
/* VQ is broken, just return and ignore any other kicks */
|
||||
return;
|
||||
}
|
||||
vhost_svq_kick(svq);
|
||||
}
|
||||
|
||||
virtio_queue_set_notification(svq->vq, true);
|
||||
} while (!virtio_queue_empty(svq->vq));
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle guest's kick.
|
||||
*
|
||||
* @n: guest kick event notifier, the one that guest set to notify svq.
|
||||
*/
|
||||
static void vhost_handle_guest_kick_notifier(EventNotifier *n)
|
||||
{
|
||||
VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
|
||||
event_notifier_test_and_clear(n);
|
||||
vhost_handle_guest_kick(svq);
|
||||
}
|
||||
|
||||
static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
|
||||
{
|
||||
if (svq->last_used_idx != svq->shadow_used_idx) {
|
||||
return true;
|
||||
}
|
||||
|
||||
svq->shadow_used_idx = cpu_to_le16(svq->vring.used->idx);
|
||||
|
||||
return svq->last_used_idx != svq->shadow_used_idx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable vhost device calls after disable them.
|
||||
*
|
||||
* @svq: The svq
|
||||
*
|
||||
* It returns false if there are pending used buffers from the vhost device,
|
||||
* avoiding the possible races between SVQ checking for more work and enabling
|
||||
* callbacks. True if SVQ used vring has no more pending buffers.
|
||||
*/
|
||||
static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
|
||||
{
|
||||
svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
|
||||
/* Make sure the flag is written before the read of used_idx */
|
||||
smp_mb();
|
||||
return !vhost_svq_more_used(svq);
|
||||
}
|
||||
|
||||
static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
|
||||
{
|
||||
svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
|
||||
}
|
||||
|
||||
static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
|
||||
uint32_t *len)
|
||||
{
|
||||
vring_desc_t *descs = svq->vring.desc;
|
||||
const vring_used_t *used = svq->vring.used;
|
||||
vring_used_elem_t used_elem;
|
||||
uint16_t last_used;
|
||||
|
||||
if (!vhost_svq_more_used(svq)) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Only get used array entries after they have been exposed by dev */
|
||||
smp_rmb();
|
||||
last_used = svq->last_used_idx & (svq->vring.num - 1);
|
||||
used_elem.id = le32_to_cpu(used->ring[last_used].id);
|
||||
used_elem.len = le32_to_cpu(used->ring[last_used].len);
|
||||
|
||||
svq->last_used_idx++;
|
||||
if (unlikely(used_elem.id >= svq->vring.num)) {
|
||||
qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
|
||||
svq->vdev->name, used_elem.id);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (unlikely(!svq->ring_id_maps[used_elem.id])) {
|
||||
qemu_log_mask(LOG_GUEST_ERROR,
|
||||
"Device %s says index %u is used, but it was not available",
|
||||
svq->vdev->name, used_elem.id);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
descs[used_elem.id].next = svq->free_head;
|
||||
svq->free_head = used_elem.id;
|
||||
|
||||
*len = used_elem.len;
|
||||
return g_steal_pointer(&svq->ring_id_maps[used_elem.id]);
|
||||
}
|
||||
|
||||
static void vhost_svq_flush(VhostShadowVirtqueue *svq,
|
||||
bool check_for_avail_queue)
|
||||
{
|
||||
VirtQueue *vq = svq->vq;
|
||||
|
||||
/* Forward as many used buffers as possible. */
|
||||
do {
|
||||
unsigned i = 0;
|
||||
|
||||
vhost_svq_disable_notification(svq);
|
||||
while (true) {
|
||||
uint32_t len;
|
||||
g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
|
||||
if (!elem) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (unlikely(i >= svq->vring.num)) {
|
||||
qemu_log_mask(LOG_GUEST_ERROR,
|
||||
"More than %u used buffers obtained in a %u size SVQ",
|
||||
i, svq->vring.num);
|
||||
virtqueue_fill(vq, elem, len, i);
|
||||
virtqueue_flush(vq, i);
|
||||
return;
|
||||
}
|
||||
virtqueue_fill(vq, elem, len, i++);
|
||||
}
|
||||
|
||||
virtqueue_flush(vq, i);
|
||||
event_notifier_set(&svq->svq_call);
|
||||
|
||||
if (check_for_avail_queue && svq->next_guest_avail_elem) {
|
||||
/*
|
||||
* Avail ring was full when vhost_svq_flush was called, so it's a
|
||||
* good moment to make more descriptors available if possible.
|
||||
*/
|
||||
vhost_handle_guest_kick(svq);
|
||||
}
|
||||
} while (!vhost_svq_enable_notification(svq));
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward used buffers.
|
||||
*
|
||||
* @n: hdev call event notifier, the one that device set to notify svq.
|
||||
*
|
||||
* Note that we are not making any buffers available in the loop, there is no
|
||||
* way that it runs more than virtqueue size times.
|
||||
*/
|
||||
static void vhost_svq_handle_call(EventNotifier *n)
|
||||
{
|
||||
VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
|
||||
hdev_call);
|
||||
event_notifier_test_and_clear(n);
|
||||
vhost_svq_flush(svq, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the call notifier for the SVQ to call the guest
|
||||
*
|
||||
* @svq: Shadow virtqueue
|
||||
* @call_fd: call notifier
|
||||
*
|
||||
* Called on BQL context.
|
||||
*/
|
||||
void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
|
||||
{
|
||||
if (call_fd == VHOST_FILE_UNBIND) {
|
||||
/*
|
||||
* Fail event_notifier_set if called handling device call.
|
||||
*
|
||||
* SVQ still needs device notifications, since it needs to keep
|
||||
* forwarding used buffers even with the unbind.
|
||||
*/
|
||||
memset(&svq->svq_call, 0, sizeof(svq->svq_call));
|
||||
} else {
|
||||
event_notifier_init_fd(&svq->svq_call, call_fd);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the shadow vq vring address.
|
||||
* @svq: Shadow virtqueue
|
||||
* @addr: Destination to store address
|
||||
*/
|
||||
void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
|
||||
struct vhost_vring_addr *addr)
|
||||
{
|
||||
addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
|
||||
addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
|
||||
addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
|
||||
}
|
||||
|
||||
size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
|
||||
{
|
||||
size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
|
||||
size_t avail_size = offsetof(vring_avail_t, ring) +
|
||||
sizeof(uint16_t) * svq->vring.num;
|
||||
|
||||
return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size);
|
||||
}
|
||||
|
||||
size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
|
||||
{
|
||||
size_t used_size = offsetof(vring_used_t, ring) +
|
||||
sizeof(vring_used_elem_t) * svq->vring.num;
|
||||
return ROUND_UP(used_size, qemu_real_host_page_size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a new file descriptor for the guest to kick the SVQ and notify for avail
|
||||
*
|
||||
* @svq: The svq
|
||||
* @svq_kick_fd: The svq kick fd
|
||||
*
|
||||
* Note that the SVQ will never close the old file descriptor.
|
||||
*/
|
||||
void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
|
||||
{
|
||||
EventNotifier *svq_kick = &svq->svq_kick;
|
||||
bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick);
|
||||
bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND;
|
||||
|
||||
if (poll_stop) {
|
||||
event_notifier_set_handler(svq_kick, NULL);
|
||||
}
|
||||
|
||||
/*
|
||||
* event_notifier_set_handler already checks for guest's notifications if
|
||||
* they arrive at the new file descriptor in the switch, so there is no
|
||||
* need to explicitly check for them.
|
||||
*/
|
||||
if (poll_start) {
|
||||
event_notifier_init_fd(svq_kick, svq_kick_fd);
|
||||
event_notifier_set(svq_kick);
|
||||
event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the shadow virtqueue operation.
|
||||
*
|
||||
* @svq: Shadow Virtqueue
|
||||
* @vdev: VirtIO device
|
||||
* @vq: Virtqueue to shadow
|
||||
*/
|
||||
void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
|
||||
VirtQueue *vq)
|
||||
{
|
||||
size_t desc_size, driver_size, device_size;
|
||||
|
||||
svq->next_guest_avail_elem = NULL;
|
||||
svq->shadow_avail_idx = 0;
|
||||
svq->shadow_used_idx = 0;
|
||||
svq->last_used_idx = 0;
|
||||
svq->vdev = vdev;
|
||||
svq->vq = vq;
|
||||
|
||||
svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
|
||||
driver_size = vhost_svq_driver_area_size(svq);
|
||||
device_size = vhost_svq_device_area_size(svq);
|
||||
svq->vring.desc = qemu_memalign(qemu_real_host_page_size, driver_size);
|
||||
desc_size = sizeof(vring_desc_t) * svq->vring.num;
|
||||
svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
|
||||
memset(svq->vring.desc, 0, driver_size);
|
||||
svq->vring.used = qemu_memalign(qemu_real_host_page_size, device_size);
|
||||
memset(svq->vring.used, 0, device_size);
|
||||
svq->ring_id_maps = g_new0(VirtQueueElement *, svq->vring.num);
|
||||
for (unsigned i = 0; i < svq->vring.num - 1; i++) {
|
||||
svq->vring.desc[i].next = cpu_to_le16(i + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the shadow virtqueue operation.
|
||||
* @svq: Shadow Virtqueue
|
||||
*/
|
||||
void vhost_svq_stop(VhostShadowVirtqueue *svq)
|
||||
{
|
||||
event_notifier_set_handler(&svq->svq_kick, NULL);
|
||||
g_autofree VirtQueueElement *next_avail_elem = NULL;
|
||||
|
||||
if (!svq->vq) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Send all pending used descriptors to guest */
|
||||
vhost_svq_flush(svq, false);
|
||||
|
||||
for (unsigned i = 0; i < svq->vring.num; ++i) {
|
||||
g_autofree VirtQueueElement *elem = NULL;
|
||||
elem = g_steal_pointer(&svq->ring_id_maps[i]);
|
||||
if (elem) {
|
||||
virtqueue_detach_element(svq->vq, elem, 0);
|
||||
}
|
||||
}
|
||||
|
||||
next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
|
||||
if (next_avail_elem) {
|
||||
virtqueue_detach_element(svq->vq, next_avail_elem, 0);
|
||||
}
|
||||
svq->vq = NULL;
|
||||
g_free(svq->ring_id_maps);
|
||||
qemu_vfree(svq->vring.desc);
|
||||
qemu_vfree(svq->vring.used);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates vhost shadow virtqueue, and instructs the vhost device to use the
|
||||
* shadow methods and file descriptors.
|
||||
*
|
||||
* @iova_tree: Tree to perform descriptors translations
|
||||
*
|
||||
* Returns the new virtqueue or NULL.
|
||||
*
|
||||
* In case of error, reason is reported through error_report.
|
||||
*/
|
||||
VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree)
|
||||
{
|
||||
g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
|
||||
int r;
|
||||
|
||||
r = event_notifier_init(&svq->hdev_kick, 0);
|
||||
if (r != 0) {
|
||||
error_report("Couldn't create kick event notifier: %s (%d)",
|
||||
g_strerror(errno), errno);
|
||||
goto err_init_hdev_kick;
|
||||
}
|
||||
|
||||
r = event_notifier_init(&svq->hdev_call, 0);
|
||||
if (r != 0) {
|
||||
error_report("Couldn't create call event notifier: %s (%d)",
|
||||
g_strerror(errno), errno);
|
||||
goto err_init_hdev_call;
|
||||
}
|
||||
|
||||
event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
|
||||
event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
|
||||
svq->iova_tree = iova_tree;
|
||||
return g_steal_pointer(&svq);
|
||||
|
||||
err_init_hdev_call:
|
||||
event_notifier_cleanup(&svq->hdev_kick);
|
||||
|
||||
err_init_hdev_kick:
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Free the resources of the shadow virtqueue.
|
||||
*
|
||||
* @pvq: gpointer to SVQ so it can be used by autofree functions.
|
||||
*/
|
||||
void vhost_svq_free(gpointer pvq)
|
||||
{
|
||||
VhostShadowVirtqueue *vq = pvq;
|
||||
vhost_svq_stop(vq);
|
||||
event_notifier_cleanup(&vq->hdev_kick);
|
||||
event_notifier_set_handler(&vq->hdev_call, NULL);
|
||||
event_notifier_cleanup(&vq->hdev_call);
|
||||
g_free(vq);
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* vhost shadow virtqueue
|
||||
*
|
||||
* SPDX-FileCopyrightText: Red Hat, Inc. 2021
|
||||
* SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef VHOST_SHADOW_VIRTQUEUE_H
|
||||
#define VHOST_SHADOW_VIRTQUEUE_H
|
||||
|
||||
#include "qemu/event_notifier.h"
|
||||
#include "hw/virtio/virtio.h"
|
||||
#include "standard-headers/linux/vhost_types.h"
|
||||
#include "hw/virtio/vhost-iova-tree.h"
|
||||
|
||||
/* Shadow virtqueue to relay notifications */
|
||||
typedef struct VhostShadowVirtqueue {
|
||||
/* Shadow vring */
|
||||
struct vring vring;
|
||||
|
||||
/* Shadow kick notifier, sent to vhost */
|
||||
EventNotifier hdev_kick;
|
||||
/* Shadow call notifier, sent to vhost */
|
||||
EventNotifier hdev_call;
|
||||
|
||||
/*
|
||||
* Borrowed virtqueue's guest to host notifier. To borrow it in this event
|
||||
* notifier allows to recover the VhostShadowVirtqueue from the event loop
|
||||
* easily. If we use the VirtQueue's one, we don't have an easy way to
|
||||
* retrieve VhostShadowVirtqueue.
|
||||
*
|
||||
* So shadow virtqueue must not clean it, or we would lose VirtQueue one.
|
||||
*/
|
||||
EventNotifier svq_kick;
|
||||
|
||||
/* Guest's call notifier, where the SVQ calls guest. */
|
||||
EventNotifier svq_call;
|
||||
|
||||
/* Virtio queue shadowing */
|
||||
VirtQueue *vq;
|
||||
|
||||
/* Virtio device */
|
||||
VirtIODevice *vdev;
|
||||
|
||||
/* IOVA mapping */
|
||||
VhostIOVATree *iova_tree;
|
||||
|
||||
/* Map for use the guest's descriptors */
|
||||
VirtQueueElement **ring_id_maps;
|
||||
|
||||
/* Next VirtQueue element that guest made available */
|
||||
VirtQueueElement *next_guest_avail_elem;
|
||||
|
||||
/* Next head to expose to the device */
|
||||
uint16_t shadow_avail_idx;
|
||||
|
||||
/* Next free descriptor */
|
||||
uint16_t free_head;
|
||||
|
||||
/* Last seen used idx */
|
||||
uint16_t shadow_used_idx;
|
||||
|
||||
/* Next head to consume from the device */
|
||||
uint16_t last_used_idx;
|
||||
} VhostShadowVirtqueue;
|
||||
|
||||
bool vhost_svq_valid_features(uint64_t features, Error **errp);
|
||||
|
||||
void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
|
||||
void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd);
|
||||
void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
|
||||
struct vhost_vring_addr *addr);
|
||||
size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq);
|
||||
size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq);
|
||||
|
||||
void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
|
||||
VirtQueue *vq);
|
||||
void vhost_svq_stop(VhostShadowVirtqueue *svq);
|
||||
|
||||
VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree);
|
||||
|
||||
void vhost_svq_free(gpointer vq);
|
||||
G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostShadowVirtqueue, vhost_svq_free);
|
||||
|
||||
#endif
|
||||
|
|
@ -17,12 +17,14 @@
|
|||
#include "hw/virtio/vhost.h"
|
||||
#include "hw/virtio/vhost-backend.h"
|
||||
#include "hw/virtio/virtio-net.h"
|
||||
#include "hw/virtio/vhost-shadow-virtqueue.h"
|
||||
#include "hw/virtio/vhost-vdpa.h"
|
||||
#include "exec/address-spaces.h"
|
||||
#include "qemu/main-loop.h"
|
||||
#include "cpu.h"
|
||||
#include "trace.h"
|
||||
#include "qemu-common.h"
|
||||
#include "qapi/error.h"
|
||||
|
||||
/*
|
||||
* Return one past the end of the end of section. Be careful with uint64_t
|
||||
|
|
@ -207,6 +209,21 @@ static void vhost_vdpa_listener_region_add(MemoryListener *listener,
|
|||
vaddr, section->readonly);
|
||||
|
||||
llsize = int128_sub(llend, int128_make64(iova));
|
||||
if (v->shadow_vqs_enabled) {
|
||||
DMAMap mem_region = {
|
||||
.translated_addr = (hwaddr)(uintptr_t)vaddr,
|
||||
.size = int128_get64(llsize) - 1,
|
||||
.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
|
||||
};
|
||||
|
||||
int r = vhost_iova_tree_map_alloc(v->iova_tree, &mem_region);
|
||||
if (unlikely(r != IOVA_OK)) {
|
||||
error_report("Can't allocate a mapping (%d)", r);
|
||||
goto fail;
|
||||
}
|
||||
|
||||
iova = mem_region.iova;
|
||||
}
|
||||
|
||||
vhost_vdpa_iotlb_batch_begin_once(v);
|
||||
ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
|
||||
|
|
@ -259,6 +276,20 @@ static void vhost_vdpa_listener_region_del(MemoryListener *listener,
|
|||
|
||||
llsize = int128_sub(llend, int128_make64(iova));
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
const DMAMap *result;
|
||||
const void *vaddr = memory_region_get_ram_ptr(section->mr) +
|
||||
section->offset_within_region +
|
||||
(iova - section->offset_within_address_space);
|
||||
DMAMap mem_region = {
|
||||
.translated_addr = (hwaddr)(uintptr_t)vaddr,
|
||||
.size = int128_get64(llsize) - 1,
|
||||
};
|
||||
|
||||
result = vhost_iova_tree_find_iova(v->iova_tree, &mem_region);
|
||||
iova = result->iova;
|
||||
vhost_iova_tree_remove(v->iova_tree, &mem_region);
|
||||
}
|
||||
vhost_vdpa_iotlb_batch_begin_once(v);
|
||||
ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
|
||||
if (ret) {
|
||||
|
|
@ -342,6 +373,55 @@ static bool vhost_vdpa_one_time_request(struct vhost_dev *dev)
|
|||
return v->index != 0;
|
||||
}
|
||||
|
||||
static int vhost_vdpa_get_dev_features(struct vhost_dev *dev,
|
||||
uint64_t *features)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
|
||||
trace_vhost_vdpa_get_features(dev, *features);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
|
||||
Error **errp)
|
||||
{
|
||||
g_autoptr(GPtrArray) shadow_vqs = NULL;
|
||||
uint64_t dev_features, svq_features;
|
||||
int r;
|
||||
bool ok;
|
||||
|
||||
if (!v->shadow_vqs_enabled) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
r = vhost_vdpa_get_dev_features(hdev, &dev_features);
|
||||
if (r != 0) {
|
||||
error_setg_errno(errp, -r, "Can't get vdpa device features");
|
||||
return r;
|
||||
}
|
||||
|
||||
svq_features = dev_features;
|
||||
ok = vhost_svq_valid_features(svq_features, errp);
|
||||
if (unlikely(!ok)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
|
||||
for (unsigned n = 0; n < hdev->nvqs; ++n) {
|
||||
g_autoptr(VhostShadowVirtqueue) svq = vhost_svq_new(v->iova_tree);
|
||||
|
||||
if (unlikely(!svq)) {
|
||||
error_setg(errp, "Cannot create svq %u", n);
|
||||
return -1;
|
||||
}
|
||||
g_ptr_array_add(shadow_vqs, g_steal_pointer(&svq));
|
||||
}
|
||||
|
||||
v->shadow_vqs = g_steal_pointer(&shadow_vqs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
|
||||
{
|
||||
struct vhost_vdpa *v;
|
||||
|
|
@ -364,6 +444,10 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
|
|||
dev->opaque = opaque ;
|
||||
v->listener = vhost_vdpa_memory_listener;
|
||||
v->msg_type = VHOST_IOTLB_MSG_V2;
|
||||
ret = vhost_vdpa_init_svq(dev, v, errp);
|
||||
if (ret) {
|
||||
goto err;
|
||||
}
|
||||
|
||||
vhost_vdpa_get_iova_range(v);
|
||||
|
||||
|
|
@ -375,6 +459,10 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
|
|||
VIRTIO_CONFIG_S_DRIVER);
|
||||
|
||||
return 0;
|
||||
|
||||
err:
|
||||
ram_block_discard_disable(false);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
|
||||
|
|
@ -445,8 +533,14 @@ static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
|
|||
|
||||
static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
int i;
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
/* FIXME SVQ is not compatible with host notifiers mr */
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) {
|
||||
if (vhost_vdpa_host_notifier_init(dev, i)) {
|
||||
goto err;
|
||||
|
|
@ -460,6 +554,21 @@ err:
|
|||
return;
|
||||
}
|
||||
|
||||
static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
size_t idx;
|
||||
|
||||
if (!v->shadow_vqs) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (idx = 0; idx < v->shadow_vqs->len; ++idx) {
|
||||
vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx));
|
||||
}
|
||||
g_ptr_array_free(v->shadow_vqs, true);
|
||||
}
|
||||
|
||||
static int vhost_vdpa_cleanup(struct vhost_dev *dev)
|
||||
{
|
||||
struct vhost_vdpa *v;
|
||||
|
|
@ -468,6 +577,7 @@ static int vhost_vdpa_cleanup(struct vhost_dev *dev)
|
|||
trace_vhost_vdpa_cleanup(dev, v);
|
||||
vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
|
||||
memory_listener_unregister(&v->listener);
|
||||
vhost_vdpa_svq_cleanup(dev);
|
||||
|
||||
dev->opaque = NULL;
|
||||
ram_block_discard_disable(false);
|
||||
|
|
@ -510,12 +620,29 @@ static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
|
|||
static int vhost_vdpa_set_features(struct vhost_dev *dev,
|
||||
uint64_t features)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
int ret;
|
||||
|
||||
if (vhost_vdpa_one_time_request(dev)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) {
|
||||
/*
|
||||
* QEMU is just trying to enable or disable logging. SVQ handles
|
||||
* this sepparately, so no need to forward this.
|
||||
*/
|
||||
v->acked_features = features;
|
||||
return 0;
|
||||
}
|
||||
|
||||
v->acked_features = features;
|
||||
|
||||
/* We must not ack _F_LOG if SVQ is enabled */
|
||||
features &= ~BIT_ULL(VHOST_F_LOG_ALL);
|
||||
}
|
||||
|
||||
trace_vhost_vdpa_set_features(dev, features);
|
||||
ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
|
||||
if (ret) {
|
||||
|
|
@ -559,11 +686,26 @@ static int vhost_vdpa_get_device_id(struct vhost_dev *dev,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static void vhost_vdpa_reset_svq(struct vhost_vdpa *v)
|
||||
{
|
||||
if (!v->shadow_vqs_enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
|
||||
VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
|
||||
vhost_svq_stop(svq);
|
||||
}
|
||||
}
|
||||
|
||||
static int vhost_vdpa_reset_device(struct vhost_dev *dev)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
int ret;
|
||||
uint8_t status = 0;
|
||||
|
||||
vhost_vdpa_reset_svq(v);
|
||||
|
||||
ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
|
||||
trace_vhost_vdpa_reset_device(dev, status);
|
||||
return ret;
|
||||
|
|
@ -647,15 +789,311 @@ static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev,
|
||||
struct vhost_vring_state *ring)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
|
||||
}
|
||||
|
||||
static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
|
||||
struct vhost_vring_file *file)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
|
||||
}
|
||||
|
||||
static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
|
||||
struct vhost_vring_file *file)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
|
||||
}
|
||||
|
||||
static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
|
||||
struct vhost_vring_addr *addr)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
|
||||
addr->desc_user_addr, addr->used_user_addr,
|
||||
addr->avail_user_addr,
|
||||
addr->log_guest_addr);
|
||||
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the shadow virtqueue descriptors to the device
|
||||
*
|
||||
* @dev: The vhost device model
|
||||
* @svq: The shadow virtqueue
|
||||
* @idx: The index of the virtqueue in the vhost device
|
||||
* @errp: Error
|
||||
*
|
||||
* Note that this function does not rewind kick file descriptor if cannot set
|
||||
* call one.
|
||||
*/
|
||||
static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
|
||||
VhostShadowVirtqueue *svq, unsigned idx,
|
||||
Error **errp)
|
||||
{
|
||||
struct vhost_vring_file file = {
|
||||
.index = dev->vq_index + idx,
|
||||
};
|
||||
const EventNotifier *event_notifier = &svq->hdev_kick;
|
||||
int r;
|
||||
|
||||
file.fd = event_notifier_get_fd(event_notifier);
|
||||
r = vhost_vdpa_set_vring_dev_kick(dev, &file);
|
||||
if (unlikely(r != 0)) {
|
||||
error_setg_errno(errp, -r, "Can't set device kick fd");
|
||||
return r;
|
||||
}
|
||||
|
||||
event_notifier = &svq->hdev_call;
|
||||
file.fd = event_notifier_get_fd(event_notifier);
|
||||
r = vhost_vdpa_set_vring_dev_call(dev, &file);
|
||||
if (unlikely(r != 0)) {
|
||||
error_setg_errno(errp, -r, "Can't set device call fd");
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unmap a SVQ area in the device
|
||||
*/
|
||||
static bool vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v,
|
||||
const DMAMap *needle)
|
||||
{
|
||||
const DMAMap *result = vhost_iova_tree_find_iova(v->iova_tree, needle);
|
||||
hwaddr size;
|
||||
int r;
|
||||
|
||||
if (unlikely(!result)) {
|
||||
error_report("Unable to find SVQ address to unmap");
|
||||
return false;
|
||||
}
|
||||
|
||||
size = ROUND_UP(result->size, qemu_real_host_page_size);
|
||||
r = vhost_vdpa_dma_unmap(v, result->iova, size);
|
||||
return r == 0;
|
||||
}
|
||||
|
||||
static bool vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
|
||||
const VhostShadowVirtqueue *svq)
|
||||
{
|
||||
DMAMap needle = {};
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
struct vhost_vring_addr svq_addr;
|
||||
bool ok;
|
||||
|
||||
vhost_svq_get_vring_addr(svq, &svq_addr);
|
||||
|
||||
needle.translated_addr = svq_addr.desc_user_addr;
|
||||
ok = vhost_vdpa_svq_unmap_ring(v, &needle);
|
||||
if (unlikely(!ok)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
needle.translated_addr = svq_addr.used_user_addr;
|
||||
return vhost_vdpa_svq_unmap_ring(v, &needle);
|
||||
}
|
||||
|
||||
/**
|
||||
* Map the SVQ area in the device
|
||||
*
|
||||
* @v: Vhost-vdpa device
|
||||
* @needle: The area to search iova
|
||||
* @errorp: Error pointer
|
||||
*/
|
||||
static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle,
|
||||
Error **errp)
|
||||
{
|
||||
int r;
|
||||
|
||||
r = vhost_iova_tree_map_alloc(v->iova_tree, needle);
|
||||
if (unlikely(r != IOVA_OK)) {
|
||||
error_setg(errp, "Cannot allocate iova (%d)", r);
|
||||
return false;
|
||||
}
|
||||
|
||||
r = vhost_vdpa_dma_map(v, needle->iova, needle->size + 1,
|
||||
(void *)(uintptr_t)needle->translated_addr,
|
||||
needle->perm == IOMMU_RO);
|
||||
if (unlikely(r != 0)) {
|
||||
error_setg_errno(errp, -r, "Cannot map region to device");
|
||||
vhost_iova_tree_remove(v->iova_tree, needle);
|
||||
}
|
||||
|
||||
return r == 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map the shadow virtqueue rings in the device
|
||||
*
|
||||
* @dev: The vhost device
|
||||
* @svq: The shadow virtqueue
|
||||
* @addr: Assigned IOVA addresses
|
||||
* @errp: Error pointer
|
||||
*/
|
||||
static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
|
||||
const VhostShadowVirtqueue *svq,
|
||||
struct vhost_vring_addr *addr,
|
||||
Error **errp)
|
||||
{
|
||||
DMAMap device_region, driver_region;
|
||||
struct vhost_vring_addr svq_addr;
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
size_t device_size = vhost_svq_device_area_size(svq);
|
||||
size_t driver_size = vhost_svq_driver_area_size(svq);
|
||||
size_t avail_offset;
|
||||
bool ok;
|
||||
|
||||
ERRP_GUARD();
|
||||
vhost_svq_get_vring_addr(svq, &svq_addr);
|
||||
|
||||
driver_region = (DMAMap) {
|
||||
.translated_addr = svq_addr.desc_user_addr,
|
||||
.size = driver_size - 1,
|
||||
.perm = IOMMU_RO,
|
||||
};
|
||||
ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp);
|
||||
if (unlikely(!ok)) {
|
||||
error_prepend(errp, "Cannot create vq driver region: ");
|
||||
return false;
|
||||
}
|
||||
addr->desc_user_addr = driver_region.iova;
|
||||
avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr;
|
||||
addr->avail_user_addr = driver_region.iova + avail_offset;
|
||||
|
||||
device_region = (DMAMap) {
|
||||
.translated_addr = svq_addr.used_user_addr,
|
||||
.size = device_size - 1,
|
||||
.perm = IOMMU_RW,
|
||||
};
|
||||
ok = vhost_vdpa_svq_map_ring(v, &device_region, errp);
|
||||
if (unlikely(!ok)) {
|
||||
error_prepend(errp, "Cannot create vq device region: ");
|
||||
vhost_vdpa_svq_unmap_ring(v, &driver_region);
|
||||
}
|
||||
addr->used_user_addr = device_region.iova;
|
||||
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
|
||||
VhostShadowVirtqueue *svq, unsigned idx,
|
||||
Error **errp)
|
||||
{
|
||||
uint16_t vq_index = dev->vq_index + idx;
|
||||
struct vhost_vring_state s = {
|
||||
.index = vq_index,
|
||||
};
|
||||
int r;
|
||||
|
||||
r = vhost_vdpa_set_dev_vring_base(dev, &s);
|
||||
if (unlikely(r)) {
|
||||
error_setg_errno(errp, -r, "Cannot set vring base");
|
||||
return false;
|
||||
}
|
||||
|
||||
r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp);
|
||||
return r == 0;
|
||||
}
|
||||
|
||||
static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
Error *err = NULL;
|
||||
unsigned i;
|
||||
|
||||
if (!v->shadow_vqs) {
|
||||
return true;
|
||||
}
|
||||
|
||||
for (i = 0; i < v->shadow_vqs->len; ++i) {
|
||||
VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i);
|
||||
VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
|
||||
struct vhost_vring_addr addr = {
|
||||
.index = i,
|
||||
};
|
||||
int r;
|
||||
bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
|
||||
if (unlikely(!ok)) {
|
||||
goto err;
|
||||
}
|
||||
|
||||
vhost_svq_start(svq, dev->vdev, vq);
|
||||
ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err);
|
||||
if (unlikely(!ok)) {
|
||||
goto err_map;
|
||||
}
|
||||
|
||||
/* Override vring GPA set by vhost subsystem */
|
||||
r = vhost_vdpa_set_vring_dev_addr(dev, &addr);
|
||||
if (unlikely(r != 0)) {
|
||||
error_setg_errno(&err, -r, "Cannot set device address");
|
||||
goto err_set_addr;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
err_set_addr:
|
||||
vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i));
|
||||
|
||||
err_map:
|
||||
vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i));
|
||||
|
||||
err:
|
||||
error_reportf_err(err, "Cannot setup SVQ %u: ", i);
|
||||
for (unsigned j = 0; j < i; ++j) {
|
||||
VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j);
|
||||
vhost_vdpa_svq_unmap_rings(dev, svq);
|
||||
vhost_svq_stop(svq);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool vhost_vdpa_svqs_stop(struct vhost_dev *dev)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
|
||||
if (!v->shadow_vqs) {
|
||||
return true;
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
|
||||
VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
|
||||
bool ok = vhost_vdpa_svq_unmap_rings(dev, svq);
|
||||
if (unlikely(!ok)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
bool ok;
|
||||
trace_vhost_vdpa_dev_start(dev, started);
|
||||
|
||||
if (started) {
|
||||
vhost_vdpa_host_notifiers_init(dev);
|
||||
ok = vhost_vdpa_svqs_start(dev);
|
||||
if (unlikely(!ok)) {
|
||||
return -1;
|
||||
}
|
||||
vhost_vdpa_set_vring_ready(dev);
|
||||
} else {
|
||||
ok = vhost_vdpa_svqs_stop(dev);
|
||||
if (unlikely(!ok)) {
|
||||
return -1;
|
||||
}
|
||||
vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
|
||||
}
|
||||
|
||||
|
|
@ -679,7 +1117,8 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
|
|||
static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
|
||||
struct vhost_log *log)
|
||||
{
|
||||
if (vhost_vdpa_one_time_request(dev)) {
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
if (v->shadow_vqs_enabled || vhost_vdpa_one_time_request(dev)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -691,11 +1130,17 @@ static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
|
|||
static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
|
||||
struct vhost_vring_addr *addr)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
|
||||
addr->desc_user_addr, addr->used_user_addr,
|
||||
addr->avail_user_addr,
|
||||
addr->log_guest_addr);
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
/*
|
||||
* Device vring addr was set at device start. SVQ base is handled by
|
||||
* VirtQueue code.
|
||||
*/
|
||||
return 0;
|
||||
}
|
||||
|
||||
return vhost_vdpa_set_vring_dev_addr(dev, addr);
|
||||
}
|
||||
|
||||
static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
|
||||
|
|
@ -708,15 +1153,41 @@ static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
|
|||
static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
|
||||
struct vhost_vring_state *ring)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
/*
|
||||
* Device vring base was set at device start. SVQ base is handled by
|
||||
* VirtQueue code.
|
||||
*/
|
||||
return 0;
|
||||
}
|
||||
|
||||
return vhost_vdpa_set_dev_vring_base(dev, ring);
|
||||
}
|
||||
|
||||
static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
|
||||
struct vhost_vring_state *ring)
|
||||
{
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
int ret;
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs,
|
||||
ring->index);
|
||||
|
||||
/*
|
||||
* Setting base as last used idx, so destination will see as available
|
||||
* all the entries that the device did not use, including the in-flight
|
||||
* processing ones.
|
||||
*
|
||||
* TODO: This is ok for networking, but other kinds of devices might
|
||||
* have problems with these retransmissions.
|
||||
*/
|
||||
ring->num = svq->last_used_idx;
|
||||
return 0;
|
||||
}
|
||||
|
||||
ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
|
||||
trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
|
||||
return ret;
|
||||
|
|
@ -725,24 +1196,45 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
|
|||
static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
|
||||
struct vhost_vring_file *file)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
int vdpa_idx = file->index - dev->vq_index;
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
|
||||
vhost_svq_set_svq_kick_fd(svq, file->fd);
|
||||
return 0;
|
||||
} else {
|
||||
return vhost_vdpa_set_vring_dev_kick(dev, file);
|
||||
}
|
||||
}
|
||||
|
||||
static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
|
||||
struct vhost_vring_file *file)
|
||||
{
|
||||
trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
|
||||
return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
|
||||
if (v->shadow_vqs_enabled) {
|
||||
int vdpa_idx = file->index - dev->vq_index;
|
||||
VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
|
||||
|
||||
vhost_svq_set_svq_call_fd(svq, file->fd);
|
||||
return 0;
|
||||
} else {
|
||||
return vhost_vdpa_set_vring_dev_call(dev, file);
|
||||
}
|
||||
}
|
||||
|
||||
static int vhost_vdpa_get_features(struct vhost_dev *dev,
|
||||
uint64_t *features)
|
||||
{
|
||||
int ret;
|
||||
struct vhost_vdpa *v = dev->opaque;
|
||||
int ret = vhost_vdpa_get_dev_features(dev, features);
|
||||
|
||||
if (ret == 0 && v->shadow_vqs_enabled) {
|
||||
/* Add SVQ logging capabilities */
|
||||
*features |= BIT_ULL(VHOST_F_LOG_ALL);
|
||||
}
|
||||
|
||||
ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
|
||||
trace_vhost_vdpa_get_features(dev, *features);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@
|
|||
#ifndef HW_VIRTIO_VHOST_VDPA_H
|
||||
#define HW_VIRTIO_VHOST_VDPA_H
|
||||
|
||||
#include <gmodule.h>
|
||||
|
||||
#include "hw/virtio/vhost-iova-tree.h"
|
||||
#include "hw/virtio/virtio.h"
|
||||
#include "standard-headers/linux/vhost_types.h"
|
||||
|
||||
|
|
@ -27,6 +30,11 @@ typedef struct vhost_vdpa {
|
|||
bool iotlb_batch_begin_sent;
|
||||
MemoryListener listener;
|
||||
struct vhost_vdpa_iova_range iova_range;
|
||||
uint64_t acked_features;
|
||||
bool shadow_vqs_enabled;
|
||||
/* IOVA mapping used by the Shadow Virtqueue */
|
||||
VhostIOVATree *iova_tree;
|
||||
GPtrArray *shadow_vqs;
|
||||
struct vhost_dev *dev;
|
||||
VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
|
||||
} VhostVDPA;
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@
|
|||
#define IOVA_OK (0)
|
||||
#define IOVA_ERR_INVALID (-1) /* Invalid parameters */
|
||||
#define IOVA_ERR_OVERLAP (-2) /* IOVA range overlapped */
|
||||
#define IOVA_ERR_NOMEM (-3) /* Cannot allocate */
|
||||
|
||||
typedef struct IOVATree IOVATree;
|
||||
typedef struct DMAMap {
|
||||
|
|
@ -82,7 +83,7 @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map);
|
|||
* @tree: the iova tree to search from
|
||||
* @map: the mapping to search
|
||||
*
|
||||
* Search for a mapping in the iova tree that overlaps with the
|
||||
* Search for a mapping in the iova tree that iova overlaps with the
|
||||
* mapping range specified. Only the first found mapping will be
|
||||
* returned.
|
||||
*
|
||||
|
|
@ -94,6 +95,24 @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map);
|
|||
*/
|
||||
const DMAMap *iova_tree_find(const IOVATree *tree, const DMAMap *map);
|
||||
|
||||
/**
|
||||
* iova_tree_find_iova:
|
||||
*
|
||||
* @tree: the iova tree to search from
|
||||
* @map: the mapping to search
|
||||
*
|
||||
* Search for a mapping in the iova tree that translated_addr overlaps with the
|
||||
* mapping range specified. Only the first found mapping will be
|
||||
* returned.
|
||||
*
|
||||
* Return: DMAMap pointer if found, or NULL if not found. Note that
|
||||
* the returned DMAMap pointer is maintained internally. User should
|
||||
* only read the content but never modify or free the content. Also,
|
||||
* user is responsible to make sure the pointer is valid (say, no
|
||||
* concurrent deletion in progress).
|
||||
*/
|
||||
const DMAMap *iova_tree_find_iova(const IOVATree *tree, const DMAMap *map);
|
||||
|
||||
/**
|
||||
* iova_tree_find_address:
|
||||
*
|
||||
|
|
@ -119,6 +138,23 @@ const DMAMap *iova_tree_find_address(const IOVATree *tree, hwaddr iova);
|
|||
*/
|
||||
void iova_tree_foreach(IOVATree *tree, iova_tree_iterator iterator);
|
||||
|
||||
/**
|
||||
* iova_tree_alloc_map:
|
||||
*
|
||||
* @tree: the iova tree to allocate from
|
||||
* @map: the new map (as translated addr & size) to allocate in the iova region
|
||||
* @iova_begin: the minimum address of the allocation
|
||||
* @iova_end: the maximum addressable direction of the allocation
|
||||
*
|
||||
* Allocates a new region of a given size, between iova_min and iova_max.
|
||||
*
|
||||
* Return: Same as iova_tree_insert, but cannot overlap and can return error if
|
||||
* iova tree is out of free contiguous range. The caller gets the assigned iova
|
||||
* in map->iova.
|
||||
*/
|
||||
int iova_tree_alloc_map(IOVATree *tree, DMAMap *map, hwaddr iova_begin,
|
||||
hwaddr iova_end);
|
||||
|
||||
/**
|
||||
* iova_tree_destroy:
|
||||
*
|
||||
|
|
|
|||
170
util/iova-tree.c
170
util/iova-tree.c
|
|
@ -16,6 +16,45 @@ struct IOVATree {
|
|||
GTree *tree;
|
||||
};
|
||||
|
||||
/* Args to pass to iova_tree_alloc foreach function. */
|
||||
struct IOVATreeAllocArgs {
|
||||
/* Size of the desired allocation */
|
||||
size_t new_size;
|
||||
|
||||
/* The minimum address allowed in the allocation */
|
||||
hwaddr iova_begin;
|
||||
|
||||
/* Map at the left of the hole, can be NULL if "this" is first one */
|
||||
const DMAMap *prev;
|
||||
|
||||
/* Map at the right of the hole, can be NULL if "prev" is the last one */
|
||||
const DMAMap *this;
|
||||
|
||||
/* If found, we fill in the IOVA here */
|
||||
hwaddr iova_result;
|
||||
|
||||
/* Whether have we found a valid IOVA */
|
||||
bool iova_found;
|
||||
};
|
||||
|
||||
typedef struct IOVATreeFindIOVAArgs {
|
||||
const DMAMap *needle;
|
||||
const DMAMap *result;
|
||||
} IOVATreeFindIOVAArgs;
|
||||
|
||||
/**
|
||||
* Iterate args to the next hole
|
||||
*
|
||||
* @args: The alloc arguments
|
||||
* @next: The next mapping in the tree. Can be NULL to signal the last one
|
||||
*/
|
||||
static void iova_tree_alloc_args_iterate(struct IOVATreeAllocArgs *args,
|
||||
const DMAMap *next)
|
||||
{
|
||||
args->prev = args->this;
|
||||
args->this = next;
|
||||
}
|
||||
|
||||
static int iova_tree_compare(gconstpointer a, gconstpointer b, gpointer data)
|
||||
{
|
||||
const DMAMap *m1 = a, *m2 = b;
|
||||
|
|
@ -47,6 +86,35 @@ const DMAMap *iova_tree_find(const IOVATree *tree, const DMAMap *map)
|
|||
return g_tree_lookup(tree->tree, map);
|
||||
}
|
||||
|
||||
static gboolean iova_tree_find_address_iterator(gpointer key, gpointer value,
|
||||
gpointer data)
|
||||
{
|
||||
const DMAMap *map = key;
|
||||
IOVATreeFindIOVAArgs *args = data;
|
||||
const DMAMap *needle;
|
||||
|
||||
g_assert(key == value);
|
||||
|
||||
needle = args->needle;
|
||||
if (map->translated_addr + map->size < needle->translated_addr ||
|
||||
needle->translated_addr + needle->size < map->translated_addr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
args->result = map;
|
||||
return true;
|
||||
}
|
||||
|
||||
const DMAMap *iova_tree_find_iova(const IOVATree *tree, const DMAMap *map)
|
||||
{
|
||||
IOVATreeFindIOVAArgs args = {
|
||||
.needle = map,
|
||||
};
|
||||
|
||||
g_tree_foreach(tree->tree, iova_tree_find_address_iterator, &args);
|
||||
return args.result;
|
||||
}
|
||||
|
||||
const DMAMap *iova_tree_find_address(const IOVATree *tree, hwaddr iova)
|
||||
{
|
||||
const DMAMap map = { .iova = iova, .size = 0 };
|
||||
|
|
@ -107,6 +175,108 @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map)
|
|||
return IOVA_OK;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to find an unallocated IOVA range between prev and this elements.
|
||||
*
|
||||
* @args: Arguments to allocation
|
||||
*
|
||||
* Cases:
|
||||
*
|
||||
* (1) !prev, !this: No entries allocated, always succeed
|
||||
*
|
||||
* (2) !prev, this: We're iterating at the 1st element.
|
||||
*
|
||||
* (3) prev, !this: We're iterating at the last element.
|
||||
*
|
||||
* (4) prev, this: this is the most common case, we'll try to find a hole
|
||||
* between "prev" and "this" mapping.
|
||||
*
|
||||
* Note that this function assumes the last valid iova is HWADDR_MAX, but it
|
||||
* searches linearly so it's easy to discard the result if it's not the case.
|
||||
*/
|
||||
static void iova_tree_alloc_map_in_hole(struct IOVATreeAllocArgs *args)
|
||||
{
|
||||
const DMAMap *prev = args->prev, *this = args->this;
|
||||
uint64_t hole_start, hole_last;
|
||||
|
||||
if (this && this->iova + this->size < args->iova_begin) {
|
||||
return;
|
||||
}
|
||||
|
||||
hole_start = MAX(prev ? prev->iova + prev->size + 1 : 0, args->iova_begin);
|
||||
hole_last = this ? this->iova : HWADDR_MAX;
|
||||
|
||||
if (hole_last - hole_start > args->new_size) {
|
||||
args->iova_result = hole_start;
|
||||
args->iova_found = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Foreach dma node in the tree, compare if there is a hole with its previous
|
||||
* node (or minimum iova address allowed) and the node.
|
||||
*
|
||||
* @key: Node iterating
|
||||
* @value: Node iterating
|
||||
* @pargs: Struct to communicate with the outside world
|
||||
*
|
||||
* Return: false to keep iterating, true if needs break.
|
||||
*/
|
||||
static gboolean iova_tree_alloc_traverse(gpointer key, gpointer value,
|
||||
gpointer pargs)
|
||||
{
|
||||
struct IOVATreeAllocArgs *args = pargs;
|
||||
DMAMap *node = value;
|
||||
|
||||
assert(key == value);
|
||||
|
||||
iova_tree_alloc_args_iterate(args, node);
|
||||
iova_tree_alloc_map_in_hole(args);
|
||||
return args->iova_found;
|
||||
}
|
||||
|
||||
int iova_tree_alloc_map(IOVATree *tree, DMAMap *map, hwaddr iova_begin,
|
||||
hwaddr iova_last)
|
||||
{
|
||||
struct IOVATreeAllocArgs args = {
|
||||
.new_size = map->size,
|
||||
.iova_begin = iova_begin,
|
||||
};
|
||||
|
||||
if (unlikely(iova_last < iova_begin)) {
|
||||
return IOVA_ERR_INVALID;
|
||||
}
|
||||
|
||||
/*
|
||||
* Find a valid hole for the mapping
|
||||
*
|
||||
* Assuming low iova_begin, so no need to do a binary search to
|
||||
* locate the first node.
|
||||
*
|
||||
* TODO: Replace all this with g_tree_node_first/next/last when available
|
||||
* (from glib since 2.68). To do it with g_tree_foreach complicates the
|
||||
* code a lot.
|
||||
*
|
||||
*/
|
||||
g_tree_foreach(tree->tree, iova_tree_alloc_traverse, &args);
|
||||
if (!args.iova_found) {
|
||||
/*
|
||||
* Either tree is empty or the last hole is still not checked.
|
||||
* g_tree_foreach does not compare (last, iova_last] range, so we check
|
||||
* it here.
|
||||
*/
|
||||
iova_tree_alloc_args_iterate(&args, NULL);
|
||||
iova_tree_alloc_map_in_hole(&args);
|
||||
}
|
||||
|
||||
if (!args.iova_found || args.iova_result + map->size > iova_last) {
|
||||
return IOVA_ERR_NOMEM;
|
||||
}
|
||||
|
||||
map->iova = args.iova_result;
|
||||
return iova_tree_insert(tree, map);
|
||||
}
|
||||
|
||||
void iova_tree_destroy(IOVATree *tree)
|
||||
{
|
||||
g_tree_destroy(tree->tree);
|
||||
|
|
|
|||
Loading…
Reference in New Issue