Re: [PATCH v6 3/3] x86/ioreq: Extend ioreq server to support multiple ioreq pages

["Teddy Astie" <teddy.astie@xxxxxxxxxx>]

Le 20/04/2026 à 11:41, Julian Vetter a écrit :
> As the number of vCPUs grows, a single ioreq page of 128 slots may not
> be sufficient. Add support for allocating and mapping multiple ioreq
> pages so that the ioreq region can scale with d->max_vcpus.
>
> Introduce nr_ioreq_pages() to compute the number of pages required for
> a given domain, and IOREQ_NR_PAGES_MAX as a compile-time upper bound
> (based on HVM_MAX_VCPUS).
>
> ioreq_server_alloc_mfn() is updated to allocate nr_ioreq_pages() pages
> and map them contiguously via vmap().
>
> is_ioreq_server_page() iterates over all ioreq pages when checking
> page ownership. ioreq_server_get_frame() allows callers to retrieve any
> ioreq page by index via the XENMEM_acquire_resource interface.
>
> On x86, the legacy GFN mapping path (hvm_map_ioreq_gfn) is limited to
> a single ioreq page; device models requiring more ioreq slots must use
> the resource mapping interface (XENMEM_acquire_resource).
>
> Signed-off-by: Julian Vetter <julian.vetter@xxxxxxxxxx>
> ---
> Changes in v6:
> - Adapted the comment to not mention the guest, but the device model
> - Replaced the dynamic allocation for the mfns array by a static array
> - Fixed error handling in ioreq_server_alloc_mfn, using an extra
>    nr_alloc variable to track the already allocated pages
> - Dropped unnecessary void casts
> ---
>   xen/arch/x86/hvm/ioreq.c |  8 ++++
>   xen/common/ioreq.c       | 93 ++++++++++++++++++++++++++++------------
>   xen/include/xen/ioreq.h  | 12 ++++++
>   3 files changed, 86 insertions(+), 27 deletions(-)
>
> diff --git a/xen/arch/x86/hvm/ioreq.c b/xen/arch/x86/hvm/ioreq.c
> index 3cabec141c..ee679bdf5a 100644
> --- a/xen/arch/x86/hvm/ioreq.c
> +++ b/xen/arch/x86/hvm/ioreq.c
> @@ -166,6 +166,14 @@ static int hvm_map_ioreq_gfn(struct ioreq_server *s, 
> bool buf)
>       if ( d->is_dying )
>           return -EINVAL;
>
> +    /*
> +     * The legacy GFN path supports only a single ioreq page. Device models
> +     * requiring more ioreq slots must use the resource mapping interface
> +     * (XENMEM_acquire_resource).
> +     */
> +    if ( !buf && nr_ioreq_pages(d) > 1 )
> +        return -EOPNOTSUPP;
> +
>       iorp->gfn = hvm_alloc_ioreq_gfn(s);
>
>       if ( gfn_eq(iorp->gfn, INVALID_GFN) )
> diff --git a/xen/common/ioreq.c b/xen/common/ioreq.c
> index bae9b99c99..3a08e77597 100644
> --- a/xen/common/ioreq.c
> +++ b/xen/common/ioreq.c
> @@ -261,8 +261,11 @@ bool vcpu_ioreq_handle_completion(struct vcpu *v)
>   static int ioreq_server_alloc_mfn(struct ioreq_server *s, bool buf)
>   {
>       struct ioreq_page *iorp = buf ? &s->bufioreq : &s->ioreq;
> -    struct page_info *page;
> -    mfn_t mfn;
> +    unsigned int i, nr_alloc = 0, nr_pages = buf ? 1 : 
> nr_ioreq_pages(s->target);
> +    mfn_t mfns[IOREQ_NR_PAGES_MAX] = {};
> +    int rc;
> +
> +    ASSERT(nr_pages <= IOREQ_NR_PAGES_MAX);
>
>       if ( iorp->va )
>       {
> @@ -277,11 +280,16 @@ static int ioreq_server_alloc_mfn(struct ioreq_server 
> *s, bool buf)
>           return 0;
>       }
>
> +    for ( i = 0; i < nr_pages; i++ )
>       {
> -        page = alloc_domheap_page(s->target, MEMF_no_refcount);
> +        struct page_info *page = alloc_domheap_page(s->target,
> +                                                    MEMF_no_refcount);
>
>           if ( !page )
> -            return -ENOMEM;
> +        {
> +            rc = -ENOMEM;
> +            goto fail;
> +        }
>
>           if ( !get_page_and_type(page, s->target, PGT_writable_page) )
>           {
> @@ -290,41 +298,59 @@ static int ioreq_server_alloc_mfn(struct ioreq_server 
> *s, bool buf)
>                * here is a clear indication of something fishy going on.
>                */
>               domain_crash(s->emulator);
> -            return -ENODATA;
> +            rc = -ENODATA;
> +            goto fail;
>           }
>
> -        mfn = page_to_mfn(page);
> +        mfns[nr_alloc++] = page_to_mfn(page);
>       }
> -    iorp->va = vmap(&mfn, 1);
> +
> +    iorp->va = vmap(mfns, nr_pages);
>       if ( !iorp->va )
> +    {
> +        rc = -ENOMEM;
>           goto fail;
> +    }
>
> -    clear_page(iorp->va);
> +    memset(iorp->va, 0, nr_pages * PAGE_SIZE);
>       return 0;
>
>    fail:
> -    put_page_alloc_ref(page);
> -    put_page_and_type(page);
> +    while ( nr_alloc-- )
> +    {
> +        struct page_info *page = mfn_to_page(mfns[nr_alloc]);
> +
> +        put_page_alloc_ref(page);
> +        put_page_and_type(page);
> +    }
>
> -    return -ENOMEM;
> +    return rc;
>   }
>
>   static void ioreq_server_free_mfn(struct ioreq_server *s, bool buf)
>   {
>       struct ioreq_page *iorp = buf ? &s->bufioreq : &s->ioreq;
> -    struct page_info *page;
> +    unsigned int i, nr_pages = buf ? 1 : nr_ioreq_pages(s->target);
> +    struct page_info *pages[IOREQ_NR_PAGES_MAX];
>       void *va;
>
>       if ( !iorp->va )
>           return;
>
> +    ASSERT(nr_pages <= IOREQ_NR_PAGES_MAX);
> +
> +    for ( i = 0; i < nr_pages; i++ )
> +        pages[i] = vmap_to_page(iorp->va + i * PAGE_SIZE);
> +
>       va = iorp->va;
> -    page = vmap_to_page(va);
>       iorp->va = NULL;
>       vunmap(va);
>
> -    put_page_alloc_ref(page);
> -    put_page_and_type(page);
> +    for ( i = 0; i < nr_pages; i++ )
> +    {
> +        put_page_alloc_ref(pages[i]);
> +        put_page_and_type(pages[i]);
> +    }
>   }
>
>   bool is_ioreq_server_page(struct domain *d, const struct page_info *page)
> @@ -337,12 +363,25 @@ bool is_ioreq_server_page(struct domain *d, const 
> struct page_info *page)
>
>       FOR_EACH_IOREQ_SERVER(d, id, s)
>       {
> -        if ( (s->ioreq.va && vmap_to_page(s->ioreq.va) == page) ||
> -             (s->bufioreq.va && vmap_to_page(s->bufioreq.va) == page) )
> +        unsigned int i;
> +
> +        if ( s->bufioreq.va && vmap_to_page(s->bufioreq.va) == page )
>           {
>               found = true;
>               break;
>           }
> +
> +        for ( i = 0; i < nr_ioreq_pages(d) && s->ioreq.va; i++ )
> +        {
> +            if ( vmap_to_page(s->ioreq.va + i * PAGE_SIZE) == page )
> +            {
> +                found = true;
> +                break;
> +            }
> +        }
> +
> +        if ( found )
> +            break;
>       }
>
>       rspin_unlock(&d->ioreq_server.lock);
> @@ -816,26 +855,26 @@ int ioreq_server_get_frame(struct domain *d, ioservid_t 
> id,
>       if ( rc )
>           goto out;
>
> -    switch ( idx )
> +    if ( idx == XENMEM_resource_ioreq_server_frame_bufioreq )
>       {
> -    case XENMEM_resource_ioreq_server_frame_bufioreq:
>           rc = -ENOENT;
>           if ( !HANDLE_BUFIOREQ(s) )
>               goto out;
>
>           *mfn = vmap_to_mfn(s->bufioreq.va);
>           rc = 0;
> -        break;
> +    }
> +    else if ( idx >= XENMEM_resource_ioreq_server_frame_ioreq(0) &&
> +              idx < 
> XENMEM_resource_ioreq_server_frame_ioreq(nr_ioreq_pages(d)) )
> +    {
> +        unsigned int page_idx = idx - 
> XENMEM_resource_ioreq_server_frame_ioreq(0);
>
> -    case XENMEM_resource_ioreq_server_frame_ioreq(0):
> -        *mfn = vmap_to_mfn(s->ioreq.va);
> +        ASSERT(page_idx < nr_ioreq_pages(d));
> +        *mfn = vmap_to_mfn(s->ioreq.va + page_idx * PAGE_SIZE);
>           rc = 0;
> -        break;
> -
> -    default:
> -        rc = -EINVAL;
> -        break;
>       }
> +    else
> +        rc = -EINVAL;
>
>    out:
>       rspin_unlock(&d->ioreq_server.lock);
> diff --git a/xen/include/xen/ioreq.h b/xen/include/xen/ioreq.h
> index d63fa4729e..d2a08c2371 100644
> --- a/xen/include/xen/ioreq.h
> +++ b/xen/include/xen/ioreq.h
> @@ -35,6 +35,18 @@ struct ioreq_vcpu {
>       bool             pending;
>   };
>
> +/*
> + * Maximum number of ioreq pages, based on the maximum number
> + * of vCPUs and the number of ioreq slots per page.
> + */
> +#define IOREQ_NR_PAGES_MAX \
> +    DIV_ROUND_UP(HVM_MAX_VCPUS, PAGE_SIZE / sizeof(ioreq_t))
> +
> +static inline unsigned int nr_ioreq_pages(const struct domain *d)
> +{
> +    return DIV_ROUND_UP(d->max_vcpus, PAGE_SIZE / sizeof(ioreq_t));
> +}
> +
>   #define NR_IO_RANGE_TYPES (XEN_DMOP_IO_RANGE_PCI + 1)
>   #define MAX_NR_IO_RANGES  256
>

Is there anything that would prevent the use of alloc_domheap_pages() to
allocate a configuous set of pages at once; and vmap_contig() to map it
in one go.
That also prevents the ioreq pages from being scattered around.

We would lose a few pages by aligning the size into a order, but that
probably better than the alternatives.

That way, we would only have to keep the base gfn (or first page_info)
and size of the allocation, and don't have to use a array of mfn nor
have to reverse the vmap to track it.

Teddy


--
Teddy Astie | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech