# HG changeset patch
# User kfraser@xxxxxxxxxxxxxxxxxxxxx
# Date 1180518373 -3600
# Node ID 45f939d0c72493d237783419996bbca0132551df
# Parent 1f7a6456c330272a3cec13b31fc1ba9b4db898ec
[LINUX] gnttab: Add basic DMA tracking
This patch adds basic tracking of outstanding DMA requests on
grant table entries marked as PageForeign.
When a PageForeign struct page is about to be mapped for DMA,
we set its map count to 1 (or zero in actual value). This is
then checked for when we need to free a grant table entry early
to ensure that we don't free an entry that's currently used for
DMA.
So any entry that has been marked for DMA will not be freed early.
If the unmapping API had a struct page (which exists for the sg
case) then we could do this properly.
Signed-off-by: Herbert Xu <herbert@xxxxxxxxxxxxxxxxxxx>
---
linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c | 18 ++
linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c | 26 ++--
linux-2.6-xen-sparse/drivers/xen/core/gnttab.c | 122 ++++++++++++++++++++
linux-2.6-xen-sparse/include/xen/gnttab.h | 14 ++
4 files changed, 169 insertions(+), 11 deletions(-)
diff -r 1f7a6456c330 -r 45f939d0c724
linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c
--- a/linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c Wed May 30
10:45:44 2007 +0100
+++ b/linux-2.6-xen-sparse/arch/i386/kernel/pci-dma-xen.c Wed May 30
10:46:13 2007 +0100
@@ -15,6 +15,7 @@
#include <linux/version.h>
#include <asm/io.h>
#include <xen/balloon.h>
+#include <xen/gnttab.h>
#include <asm/swiotlb.h>
#include <asm/tlbflush.h>
#include <asm-i386/mach-xen/asm/swiotlb.h>
@@ -90,7 +91,7 @@ dma_map_sg(struct device *hwdev, struct
} else {
for (i = 0; i < nents; i++ ) {
sg[i].dma_address =
- page_to_bus(sg[i].page) + sg[i].offset;
+ gnttab_dma_map_page(sg[i].page) + sg[i].offset;
sg[i].dma_length = sg[i].length;
BUG_ON(!sg[i].page);
IOMMU_BUG_ON(address_needs_mapping(
@@ -108,9 +109,15 @@ dma_unmap_sg(struct device *hwdev, struc
dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
enum dma_data_direction direction)
{
+ int i;
+
BUG_ON(direction == DMA_NONE);
if (swiotlb)
swiotlb_unmap_sg(hwdev, sg, nents, direction);
+ else {
+ for (i = 0; i < nents; i++ )
+ gnttab_dma_unmap_page(sg[i].dma_address);
+ }
}
EXPORT_SYMBOL(dma_unmap_sg);
@@ -127,7 +134,7 @@ dma_map_page(struct device *dev, struct
dma_addr = swiotlb_map_page(
dev, page, offset, size, direction);
} else {
- dma_addr = page_to_bus(page) + offset;
+ dma_addr = gnttab_dma_map_page(page) + offset;
IOMMU_BUG_ON(address_needs_mapping(dev, dma_addr));
}
@@ -142,6 +149,8 @@ dma_unmap_page(struct device *dev, dma_a
BUG_ON(direction == DMA_NONE);
if (swiotlb)
swiotlb_unmap_page(dev, dma_address, size, direction);
+ else
+ gnttab_dma_unmap_page(dma_address);
}
EXPORT_SYMBOL(dma_unmap_page);
#endif /* CONFIG_HIGHMEM */
@@ -326,7 +335,8 @@ dma_map_single(struct device *dev, void
if (swiotlb) {
dma = swiotlb_map_single(dev, ptr, size, direction);
} else {
- dma = virt_to_bus(ptr);
+ dma = gnttab_dma_map_page(virt_to_page(ptr)) +
+ offset_in_page(ptr);
IOMMU_BUG_ON(range_straddles_page_boundary(ptr, size));
IOMMU_BUG_ON(address_needs_mapping(dev, dma));
}
@@ -344,6 +354,8 @@ dma_unmap_single(struct device *dev, dma
BUG();
if (swiotlb)
swiotlb_unmap_single(dev, dma_addr, size, direction);
+ else
+ gnttab_dma_unmap_page(dma_addr);
}
EXPORT_SYMBOL(dma_unmap_single);
diff -r 1f7a6456c330 -r 45f939d0c724
linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c
--- a/linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c Wed May 30 10:45:44
2007 +0100
+++ b/linux-2.6-xen-sparse/arch/i386/kernel/swiotlb.c Wed May 30 10:46:13
2007 +0100
@@ -25,14 +25,13 @@
#include <asm/pci.h>
#include <asm/dma.h>
#include <asm/uaccess.h>
+#include <xen/gnttab.h>
#include <xen/interface/memory.h>
int swiotlb;
EXPORT_SYMBOL(swiotlb);
#define OFFSET(val,align) ((unsigned long)((val) & ( (align) - 1)))
-
-#define SG_ENT_PHYS_ADDRESS(sg) (page_to_bus((sg)->page) + (sg)->offset)
/*
* Maximum allowable number of contiguous slabs to map,
@@ -468,7 +467,8 @@ dma_addr_t
dma_addr_t
swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
{
- dma_addr_t dev_addr = virt_to_bus(ptr);
+ dma_addr_t dev_addr = gnttab_dma_map_page(virt_to_page(ptr)) +
+ offset_in_page(ptr);
void *map;
struct phys_addr buffer;
@@ -486,6 +486,7 @@ swiotlb_map_single(struct device *hwdev,
/*
* Oh well, have to allocate and map a bounce buffer.
*/
+ gnttab_dma_unmap_page(dev_addr);
buffer.page = virt_to_page(ptr);
buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
map = map_single(hwdev, buffer, size, dir);
@@ -513,6 +514,8 @@ swiotlb_unmap_single(struct device *hwde
BUG_ON(dir == DMA_NONE);
if (in_swiotlb_aperture(dev_addr))
unmap_single(hwdev, bus_to_virt(dev_addr), size, dir);
+ else
+ gnttab_dma_unmap_page(dev_addr);
}
/*
@@ -571,8 +574,10 @@ swiotlb_map_sg(struct device *hwdev, str
BUG_ON(dir == DMA_NONE);
for (i = 0; i < nelems; i++, sg++) {
- dev_addr = SG_ENT_PHYS_ADDRESS(sg);
+ dev_addr = gnttab_dma_map_page(sg->page) + sg->offset;
+
if (address_needs_mapping(hwdev, dev_addr)) {
+ gnttab_dma_unmap_page(dev_addr);
buffer.page = sg->page;
buffer.offset = sg->offset;
map = map_single(hwdev, buffer, sg->length, dir);
@@ -605,10 +610,12 @@ swiotlb_unmap_sg(struct device *hwdev, s
BUG_ON(dir == DMA_NONE);
for (i = 0; i < nelems; i++, sg++)
- if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+ if (in_swiotlb_aperture(sg->dma_address))
unmap_single(hwdev,
(void *)bus_to_virt(sg->dma_address),
sg->dma_length, dir);
+ else
+ gnttab_dma_unmap_page(sg->dma_address);
}
/*
@@ -627,7 +634,7 @@ swiotlb_sync_sg_for_cpu(struct device *h
BUG_ON(dir == DMA_NONE);
for (i = 0; i < nelems; i++, sg++)
- if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+ if (in_swiotlb_aperture(sg->dma_address))
sync_single(hwdev,
(void *)bus_to_virt(sg->dma_address),
sg->dma_length, dir);
@@ -642,7 +649,7 @@ swiotlb_sync_sg_for_device(struct device
BUG_ON(dir == DMA_NONE);
for (i = 0; i < nelems; i++, sg++)
- if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+ if (in_swiotlb_aperture(sg->dma_address))
sync_single(hwdev,
(void *)bus_to_virt(sg->dma_address),
sg->dma_length, dir);
@@ -659,8 +666,9 @@ swiotlb_map_page(struct device *hwdev, s
dma_addr_t dev_addr;
char *map;
- dev_addr = page_to_bus(page) + offset;
+ dev_addr = gnttab_dma_map_page(page) + offset;
if (address_needs_mapping(hwdev, dev_addr)) {
+ gnttab_dma_unmap_page(dev_addr);
buffer.page = page;
buffer.offset = offset;
map = map_single(hwdev, buffer, size, direction);
@@ -681,6 +689,8 @@ swiotlb_unmap_page(struct device *hwdev,
BUG_ON(direction == DMA_NONE);
if (in_swiotlb_aperture(dma_address))
unmap_single(hwdev, bus_to_virt(dma_address), size, direction);
+ else
+ gnttab_dma_unmap_page(dma_address);
}
#endif
diff -r 1f7a6456c330 -r 45f939d0c724
linux-2.6-xen-sparse/drivers/xen/core/gnttab.c
--- a/linux-2.6-xen-sparse/drivers/xen/core/gnttab.c Wed May 30 10:45:44
2007 +0100
+++ b/linux-2.6-xen-sparse/drivers/xen/core/gnttab.c Wed May 30 10:46:13
2007 +0100
@@ -490,6 +490,128 @@ static int gnttab_map(unsigned int start
return 0;
}
+static void gnttab_page_free(struct page *page)
+{
+ if (page->mapping) {
+ put_page((struct page *)page->mapping);
+ page->mapping = NULL;
+ }
+
+ ClearPageForeign(page);
+ gnttab_reset_grant_page(page);
+ put_page(page);
+}
+
+/*
+ * Must not be called with IRQs off. This should only be used on the
+ * slow path.
+ *
+ * Copy a foreign granted page to local memory.
+ */
+int gnttab_copy_grant_page(grant_ref_t ref, struct page **pagep)
+{
+ struct gnttab_unmap_and_replace unmap;
+ mmu_update_t mmu;
+ struct page *page;
+ struct page *new_page;
+ void *new_addr;
+ void *addr;
+ paddr_t pfn;
+ maddr_t mfn;
+ maddr_t new_mfn;
+ int err;
+
+ page = *pagep;
+ if (!get_page_unless_zero(page))
+ return -ENOENT;
+
+ err = -ENOMEM;
+ new_page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
+ if (!new_page)
+ goto out;
+
+ new_addr = page_address(new_page);
+ addr = page_address(page);
+ memcpy(new_addr, addr, PAGE_SIZE);
+
+ pfn = page_to_pfn(page);
+ mfn = pfn_to_mfn(pfn);
+ new_mfn = virt_to_mfn(new_addr);
+
+ if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+ set_phys_to_machine(pfn, new_mfn);
+ set_phys_to_machine(page_to_pfn(new_page), INVALID_P2M_ENTRY);
+
+ mmu.ptr = (new_mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
+ mmu.val = pfn;
+ err = HYPERVISOR_mmu_update(&mmu, 1, NULL, DOMID_SELF);
+ BUG_ON(err);
+ }
+
+ gnttab_set_replace_op(&unmap, (unsigned long)addr,
+ (unsigned long)new_addr, ref);
+
+ err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_and_replace,
+ &unmap, 1);
+ BUG_ON(err);
+ BUG_ON(unmap.status);
+
+ new_page->mapping = page->mapping;
+ new_page->index = page->index;
+ set_bit(PG_foreign, &new_page->flags);
+ *pagep = new_page;
+
+ SetPageForeign(page, gnttab_page_free);
+ page->mapping = NULL;
+
+ /*
+ * Ensure that there is a barrier between setting the p2m entry
+ * and checking the map count. See gnttab_dma_map_page.
+ */
+ smp_mb();
+
+ /* Has the page been DMA-mapped? */
+ if (unlikely(page_mapped(page))) {
+ err = -EBUSY;
+ page->mapping = (void *)new_page;
+ }
+
+out:
+ put_page(page);
+ return err;
+
+}
+EXPORT_SYMBOL(gnttab_copy_grant_page);
+
+/*
+ * Keep track of foreign pages marked as PageForeign so that we don't
+ * return them to the remote domain prematurely.
+ *
+ * PageForeign pages are pinned down by increasing their mapcount.
+ *
+ * All other pages are simply returned as is.
+ */
+maddr_t gnttab_dma_map_page(struct page *page)
+{
+ maddr_t mfn = pfn_to_mfn(page_to_pfn(page)), mfn2;
+
+ if (!PageForeign(page))
+ return mfn << PAGE_SHIFT;
+
+ if (mfn_to_local_pfn(mfn) < max_mapnr)
+ return mfn << PAGE_SHIFT;
+
+ atomic_set(&page->_mapcount, 0);
+
+ /* This barrier corresponds to the one in gnttab_copy_grant_page. */
+ smp_mb();
+
+ /* Has this page been copied in the mean time? */
+ mfn2 = pfn_to_mfn(page_to_pfn(page));
+
+ return mfn2 << PAGE_SHIFT;
+}
+
int gnttab_resume(void)
{
if (max_nr_grant_frames() < nr_grant_frames)
diff -r 1f7a6456c330 -r 45f939d0c724 linux-2.6-xen-sparse/include/xen/gnttab.h
--- a/linux-2.6-xen-sparse/include/xen/gnttab.h Wed May 30 10:45:44 2007 +0100
+++ b/linux-2.6-xen-sparse/include/xen/gnttab.h Wed May 30 10:46:13 2007 +0100
@@ -39,6 +39,7 @@
#include <asm/hypervisor.h>
#include <asm/maddr.h> /* maddr_t */
+#include <linux/mm.h>
#include <xen/interface/grant_table.h>
#include <xen/features.h>
@@ -101,6 +102,19 @@ void gnttab_grant_foreign_transfer_ref(g
void gnttab_grant_foreign_transfer_ref(grant_ref_t, domid_t domid,
unsigned long pfn);
+int gnttab_copy_grant_page(grant_ref_t ref, struct page **pagep);
+maddr_t gnttab_dma_map_page(struct page *page);
+
+static inline void gnttab_dma_unmap_page(maddr_t mfn)
+{
+}
+
+static inline void gnttab_reset_grant_page(struct page *page)
+{
+ init_page_count(page);
+ reset_page_mapcount(page);
+}
+
int gnttab_suspend(void);
int gnttab_resume(void);
_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog
|
Home