Maksytka/linux
1
1
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2025-12-01 07:26:02 +07:00
Code Issues Packages Projects Releases Wiki Activity

io_uring/zcrx: split out memory holders from area

Browse Source 
In the data path users of struct io_zcrx_area don't need to know what
kind of memory it's backed by. Only keep there generic bits in there and
and split out memory type dependent fields into a new structure. It also
logically separates the step that actually imports the memory, e.g.
pinning user pages, from the generic area initialisation.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b60fc09c76921bf69e77eb17e07eb4decedb3bf4.1746097431.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov
2025-05-01 13:17:16 +01:00
committed by Jens Axboe
parent 6c9589aa08
commit 782dfa329a
2 changed files with 56 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
io_uring/zcrx.c
View File

@@ -26,6 +26,8 @@
#include "zcrx.h"
#include "rsrc.h"
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)
{
return pp->mp_priv;
@@ -42,10 +44,43 @@ static inline struct page *io_zcrx_iov_page(const struct net_iov *niov)
{
struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
return area->pages[net_iov_idx(niov)];
return area->mem.pages[net_iov_idx(niov)];
}
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
static void io_release_area_mem(struct io_zcrx_mem *mem)
{
if (mem->pages) {
unpin_user_pages(mem->pages, mem->nr_folios);
kvfree(mem->pages);
}
}
static int io_import_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_mem *mem,
struct io_uring_zcrx_area_reg *area_reg)
{
struct page **pages;
int nr_pages;
int ret;
ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
if (ret)
return ret;
if (!area_reg->addr)
return -EFAULT;
if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
return -EINVAL;
pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
&nr_pages);
if (IS_ERR(pages))
return PTR_ERR(pages);
mem->pages = pages;
mem->nr_folios = nr_pages;
mem->size = area_reg->len;
return 0;
}
static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_area *area, int nr_mapped)
@@ -84,8 +119,8 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
struct net_iov *niov = &area->nia.niovs[i];
dma_addr_t dma;
dma = dma_map_page_attrs(ifq->dev, area->pages[i], 0, PAGE_SIZE,
DMA_FROM_DEVICE, IO_DMA_ATTR);
dma = dma_map_page_attrs(ifq->dev, area->mem.pages[i], 0,
PAGE_SIZE, DMA_FROM_DEVICE, IO_DMA_ATTR);
if (dma_mapping_error(ifq->dev, dma))
break;
if (net_mp_niov_set_dma_addr(niov, dma)) {
@@ -192,14 +227,11 @@ static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)
static void io_zcrx_free_area(struct io_zcrx_area *area)
{
io_zcrx_unmap_area(area->ifq, area);
io_release_area_mem(&area->mem);
kvfree(area->freelist);
kvfree(area->nia.niovs);
kvfree(area->user_refs);
if (area->pages) {
unpin_user_pages(area->pages, area->nr_folios);
kvfree(area->pages);
}
kfree(area);
}
@@ -208,36 +240,27 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,
struct io_uring_zcrx_area_reg *area_reg)
{
struct io_zcrx_area *area;
int i, ret, nr_pages, nr_iovs;
unsigned nr_iovs;
int i, ret;
if (area_reg->flags || area_reg->rq_area_token)
return -EINVAL;
if (area_reg->__resv1 || area_reg->__resv2[0] || area_reg->__resv2[1])
return -EINVAL;
if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK)
return -EINVAL;
ret = io_validate_user_buf_range(area_reg->addr, area_reg->len);
if (ret)
return ret;
if (!area_reg->addr)
return -EFAULT;
ret = -ENOMEM;
area = kzalloc(sizeof(*area), GFP_KERNEL);
if (!area)
goto err;
area->pages = io_pin_pages((unsigned long)area_reg->addr, area_reg->len,
&nr_pages);
if (IS_ERR(area->pages)) {
ret = PTR_ERR(area->pages);
area->pages = NULL;
ret = io_import_area(ifq, &area->mem, area_reg);
if (ret)
goto err;
}
area->nr_folios = nr_iovs = nr_pages;
nr_iovs = area->mem.size >> PAGE_SHIFT;
area->nia.num_niovs = nr_iovs;
ret = -ENOMEM;
area->nia.niovs = kvmalloc_array(nr_iovs, sizeof(area->nia.niovs[0]),
GFP_KERNEL | __GFP_ZERO);
if (!area->nia.niovs)

11
io_uring/zcrx.h
View File

@@ -7,6 +7,13 @@
#include <net/page_pool/types.h>
#include <net/net_trackers.h>
struct io_zcrx_mem {
unsigned long size;
struct page **pages;
unsigned long nr_folios;
};
struct io_zcrx_area {
struct net_iov_area nia;
struct io_zcrx_ifq *ifq;
@@ -14,13 +21,13 @@ struct io_zcrx_area {
bool is_mapped;
u16 area_id;
struct page **pages;
unsigned long nr_folios;
/* freelist */
spinlock_t freelist_lock ____cacheline_aligned_in_smp;
u32 free_count;
u32 *freelist;
struct io_zcrx_mem mem;
};
struct io_zcrx_ifq {