# HG changeset patch
# User kaf24@xxxxxxxxxxxxxxxxxxxx
# Node ID 3d187585c1415fbb14cb285cc71e0ff0faf81e67
# Parent eaf498f1ffdef1d63ef9df03f1d8ea749227d183
Use a SWIOTLB to use pre-reserved bounce buffers for high memory
and multi-page DMA accesses. This is based on a preliminary patch
from Suresh Siddha at Intel.
Signed-off-by: Keir Fraser <keir@xxxxxxxxxxxxx>
Signed-off-by: Suresh Siddha <suresh.b.siddha@xxxxxxxxx>
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32 Tue Aug
16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32 Tue Aug
16 11:20:47 2005
@@ -130,6 +130,7 @@
# CONFIG_X86_REBOOTFIXUPS is not set
CONFIG_MICROCODE=y
CONFIG_X86_CPUID=y
+CONFIG_SWIOTLB=y
#
# Firmware Drivers
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64 Tue Aug
16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64 Tue Aug
16 11:20:47 2005
@@ -123,6 +123,7 @@
# CONFIG_X86_MSR is not set
# CONFIG_GART_IOMMU is not set
CONFIG_DUMMY_IOMMU=y
+CONFIG_SWIOTLB=y
# CONFIG_X86_MCE is not set
#
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32 Tue Aug
16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32 Tue Aug
16 11:20:47 2005
@@ -137,6 +137,7 @@
# CONFIG_X86_REBOOTFIXUPS is not set
CONFIG_MICROCODE=m
CONFIG_X86_CPUID=m
+CONFIG_SWIOTLB=y
#
# Firmware Drivers
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64
--- a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64 Tue Aug
16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64 Tue Aug
16 11:20:47 2005
@@ -132,6 +132,7 @@
# CONFIG_NUMA_EMU is not set
# CONFIG_GART_IOMMU is not set
CONFIG_DUMMY_IOMMU=y
+CONFIG_SWIOTLB=y
# CONFIG_X86_MCE is not set
#
diff -r eaf498f1ffde -r 3d187585c141 linux-2.6-xen-sparse/arch/xen/i386/Kconfig
--- a/linux-2.6-xen-sparse/arch/xen/i386/Kconfig Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/i386/Kconfig Tue Aug 16 11:20:47 2005
@@ -533,6 +533,11 @@
with major 203 and minors 0 to 31 for /dev/cpu/0/cpuid to
/dev/cpu/31/cpuid.
+config SWIOTLB
+ bool
+ depends on PCI
+ default y
+
source "drivers/firmware/Kconfig"
choice
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile
--- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile Tue Aug 16
10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile Tue Aug 16
11:20:47 2005
@@ -44,6 +44,7 @@
c-obj-$(CONFIG_EFI) += efi.o efi_stub.o
c-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
c-obj-$(CONFIG_SMP_ALTERNATIVES)+= smpalts.o
+c-obj-$(CONFIG_SWIOTLB) += swiotlb.o
EXTRA_AFLAGS := -traditional
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c
--- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c Tue Aug 16
10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c Tue Aug 16
11:20:47 2005
@@ -23,6 +23,103 @@
int flags;
unsigned long *bitmap;
};
+
+static void iommu_bug(void)
+{
+ printk(KERN_ALERT "Fatal DMA error! Please use 'swiotlb=force'\n");
+ BUG();
+}
+
+#define IOMMU_BUG_ON(test) do { if (unlikely(test)) iommu_bug(); } while(0)
+
+int
+dma_map_sg(struct device *hwdev, struct scatterlist *sg, int nents,
+ enum dma_data_direction direction)
+{
+ int i, rc;
+
+ BUG_ON(direction == DMA_NONE);
+
+ if (swiotlb) {
+ rc = swiotlb_map_sg(hwdev, sg, nents, direction);
+ } else {
+ for (i = 0; i < nents; i++ ) {
+ sg[i].dma_address =
+ page_to_phys(sg[i].page) + sg[i].offset;
+ sg[i].dma_length = sg[i].length;
+ BUG_ON(!sg[i].page);
+ IOMMU_BUG_ON(address_needs_mapping(
+ hwdev, sg[i].dma_address));
+ }
+ rc = nents;
+ }
+
+ flush_write_buffers();
+ return rc;
+}
+EXPORT_SYMBOL(dma_map_sg);
+
+void
+dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
+ enum dma_data_direction direction)
+{
+ BUG_ON(direction == DMA_NONE);
+ if (swiotlb)
+ swiotlb_unmap_sg(hwdev, sg, nents, direction);
+}
+EXPORT_SYMBOL(dma_unmap_sg);
+
+dma_addr_t
+dma_map_page(struct device *dev, struct page *page, unsigned long offset,
+ size_t size, enum dma_data_direction direction)
+{
+ dma_addr_t dma_addr;
+
+ BUG_ON(direction == DMA_NONE);
+
+ if (swiotlb) {
+ dma_addr = swiotlb_map_page(
+ dev, page, offset, size, direction);
+ } else {
+ dma_addr = page_to_phys(page) + offset;
+ IOMMU_BUG_ON(address_needs_mapping(dev, dma_addr));
+ }
+
+ return dma_addr;
+}
+EXPORT_SYMBOL(dma_map_page);
+
+void
+dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
+ enum dma_data_direction direction)
+{
+ BUG_ON(direction == DMA_NONE);
+ if (swiotlb)
+ swiotlb_unmap_page(dev, dma_address, size, direction);
+}
+EXPORT_SYMBOL(dma_unmap_page);
+
+int
+dma_mapping_error(dma_addr_t dma_addr)
+{
+ if (swiotlb)
+ return swiotlb_dma_mapping_error(dma_addr);
+ return 0;
+}
+EXPORT_SYMBOL(dma_mapping_error);
+
+int
+dma_supported(struct device *dev, u64 mask)
+{
+ if (swiotlb)
+ return swiotlb_dma_supported(dev, mask);
+ /*
+ * By default we'll BUG when an infeasible DMA is requested, and
+ * request swiotlb=force (see IOMMU_BUG_ON).
+ */
+ return 1;
+}
+EXPORT_SYMBOL(dma_supported);
void *dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, unsigned int __nocast gfp)
@@ -54,13 +151,14 @@
ret = (void *)vstart;
if (ret != NULL) {
- xen_contig_memory(vstart, order);
+ xen_create_contiguous_region(vstart, order);
memset(ret, 0, size);
*dma_handle = virt_to_bus(ret);
}
return ret;
}
+EXPORT_SYMBOL(dma_alloc_coherent);
void dma_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle)
@@ -72,9 +170,12 @@
int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
bitmap_release_region(mem->bitmap, page, order);
- } else
+ } else {
+ xen_destroy_contiguous_region((unsigned long)vaddr, order);
free_pages((unsigned long)vaddr, order);
-}
+ }
+}
+EXPORT_SYMBOL(dma_free_coherent);
int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
dma_addr_t device_addr, size_t size, int flags)
@@ -153,46 +254,20 @@
}
EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
-static LIST_HEAD(dma_map_head);
-static DEFINE_SPINLOCK(dma_map_lock);
-struct dma_map_entry {
- struct list_head list;
- dma_addr_t dma;
- char *bounce, *host;
- size_t size;
-};
-#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d)))
-
dma_addr_t
dma_map_single(struct device *dev, void *ptr, size_t size,
enum dma_data_direction direction)
{
- struct dma_map_entry *ent;
- void *bnc;
dma_addr_t dma;
- unsigned long flags;
-
- BUG_ON(direction == DMA_NONE);
-
- /*
- * Even if size is sub-page, the buffer may still straddle a page
- * boundary. Take into account buffer start offset. All other calls are
- * conservative and always search the dma_map list if it's non-empty.
- */
- if ((((unsigned int)ptr & ~PAGE_MASK) + size) <= PAGE_SIZE) {
+
+ BUG_ON(direction == DMA_NONE);
+
+ if (swiotlb) {
+ dma = swiotlb_map_single(dev, ptr, size, direction);
+ } else {
dma = virt_to_bus(ptr);
- } else {
- BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, GFP_ATOMIC))
== NULL);
- BUG_ON((ent = kmalloc(sizeof(*ent), GFP_ATOMIC)) == NULL);
- if (direction != DMA_FROM_DEVICE)
- memcpy(bnc, ptr, size);
- ent->dma = dma;
- ent->bounce = bnc;
- ent->host = ptr;
- ent->size = size;
- spin_lock_irqsave(&dma_map_lock, flags);
- list_add(&ent->list, &dma_map_head);
- spin_unlock_irqrestore(&dma_map_lock, flags);
+ IOMMU_BUG_ON(range_straddles_page_boundary(ptr, size));
+ IOMMU_BUG_ON(address_needs_mapping(dev, dma));
}
flush_write_buffers();
@@ -204,30 +279,9 @@
dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
enum dma_data_direction direction)
{
- struct dma_map_entry *ent;
- unsigned long flags;
-
- BUG_ON(direction == DMA_NONE);
-
- /* Fast-path check: are there any multi-page DMA mappings? */
- if (!list_empty(&dma_map_head)) {
- spin_lock_irqsave(&dma_map_lock, flags);
- list_for_each_entry ( ent, &dma_map_head, list ) {
- if (DMA_MAP_MATCHES(ent, dma_addr)) {
- list_del(&ent->list);
- break;
- }
- }
- spin_unlock_irqrestore(&dma_map_lock, flags);
- if (&ent->list != &dma_map_head) {
- BUG_ON(dma_addr != ent->dma);
- BUG_ON(size != ent->size);
- if (direction != DMA_TO_DEVICE)
- memcpy(ent->host, ent->bounce, size);
- dma_free_coherent(dev, size, ent->bounce, ent->dma);
- kfree(ent);
- }
- }
+ BUG_ON(direction == DMA_NONE);
+ if (swiotlb)
+ swiotlb_unmap_single(dev, dma_addr, size, direction);
}
EXPORT_SYMBOL(dma_unmap_single);
@@ -235,23 +289,8 @@
dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
enum dma_data_direction direction)
{
- struct dma_map_entry *ent;
- unsigned long flags, off;
-
- /* Fast-path check: are there any multi-page DMA mappings? */
- if (!list_empty(&dma_map_head)) {
- spin_lock_irqsave(&dma_map_lock, flags);
- list_for_each_entry ( ent, &dma_map_head, list )
- if (DMA_MAP_MATCHES(ent, dma_handle))
- break;
- spin_unlock_irqrestore(&dma_map_lock, flags);
- if (&ent->list != &dma_map_head) {
- off = dma_handle - ent->dma;
- BUG_ON((off + size) > ent->size);
- /*if (direction != DMA_TO_DEVICE)*/
- memcpy(ent->host+off, ent->bounce+off, size);
- }
- }
+ if (swiotlb)
+ swiotlb_sync_single_for_cpu(dev, dma_handle, size, direction);
}
EXPORT_SYMBOL(dma_sync_single_for_cpu);
@@ -259,24 +298,17 @@
dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t
size,
enum dma_data_direction direction)
{
- struct dma_map_entry *ent;
- unsigned long flags, off;
-
- /* Fast-path check: are there any multi-page DMA mappings? */
- if (!list_empty(&dma_map_head)) {
- spin_lock_irqsave(&dma_map_lock, flags);
- list_for_each_entry ( ent, &dma_map_head, list )
- if (DMA_MAP_MATCHES(ent, dma_handle))
- break;
- spin_unlock_irqrestore(&dma_map_lock, flags);
- if (&ent->list != &dma_map_head) {
- off = dma_handle - ent->dma;
- BUG_ON((off + size) > ent->size);
- /*if (direction != DMA_FROM_DEVICE)*/
- memcpy(ent->bounce+off, ent->host+off, size);
- }
- }
-
- flush_write_buffers();
+ if (swiotlb)
+ swiotlb_sync_single_for_device(dev, dma_handle, size,
direction);
}
EXPORT_SYMBOL(dma_sync_single_for_device);
+
+/*
+ * Local variables:
+ * c-file-style: "linux"
+ * indent-tabs-mode: t
+ * c-indent-level: 8
+ * c-basic-offset: 8
+ * tab-width: 8
+ * End:
+ */
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c
--- a/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c Tue Aug 16
10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c Tue Aug 16
11:20:47 2005
@@ -263,12 +263,9 @@
BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
}
-void xen_contig_memory(unsigned long vstart, unsigned int order)
-{
- /*
- * Ensure multi-page extents are contiguous in machine memory. This code
- * could be cleaned up some, and the number of hypercalls reduced.
- */
+/* Ensure multi-page extents are contiguous in machine memory. */
+void xen_create_contiguous_region(unsigned long vstart, unsigned int order)
+{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
@@ -312,6 +309,49 @@
balloon_unlock(flags);
}
+void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+ unsigned long mfn, i, flags;
+
+ scrub_pages(vstart, 1 << order);
+
+ balloon_lock(flags);
+
+ /* 1. Zap current PTEs, giving away the underlying pages. */
+ for (i = 0; i < (1<<order); i++) {
+ pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
+ pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
+ pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
+ pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
+ mfn = pte_mfn(*pte);
+ BUG_ON(HYPERVISOR_update_va_mapping(
+ vstart + (i*PAGE_SIZE), __pte_ma(0), 0));
+ phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
+ INVALID_P2M_ENTRY;
+ BUG_ON(HYPERVISOR_dom_mem_op(
+ MEMOP_decrease_reservation, &mfn, 1, 0) != 1);
+ }
+
+ /* 2. Map new pages in place of old pages. */
+ for (i = 0; i < (1<<order); i++) {
+ BUG_ON(HYPERVISOR_dom_mem_op(
+ MEMOP_increase_reservation, &mfn, 1, 0) != 1);
+ BUG_ON(HYPERVISOR_update_va_mapping(
+ vstart + (i*PAGE_SIZE),
+ __pte_ma((mfn<<PAGE_SHIFT)|__PAGE_KERNEL), 0));
+ xen_machphys_update(mfn, (__pa(vstart)>>PAGE_SHIFT)+i);
+ phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = mfn;
+ }
+
+ flush_tlb_all();
+
+ balloon_unlock(flags);
+}
+
#ifdef CONFIG_XEN_PHYSDEV_ACCESS
unsigned long allocate_empty_lowmem_region(unsigned long pages)
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/i386/mm/init.c
--- a/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c Tue Aug 16 11:20:47 2005
@@ -41,6 +41,12 @@
#include <asm/sections.h>
#include <asm-xen/hypervisor.h>
+#if defined(CONFIG_SWIOTLB)
+extern void swiotlb_init(void);
+int swiotlb;
+EXPORT_SYMBOL(swiotlb);
+#endif
+
unsigned int __VMALLOC_RESERVE = 128 << 20;
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -630,6 +636,10 @@
int tmp;
int bad_ppro;
unsigned long pfn;
+
+#if defined(CONFIG_SWIOTLB)
+ swiotlb_init();
+#endif
#ifndef CONFIG_DISCONTIGMEM
if (!mem_map)
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c
--- a/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c Tue Aug 16 11:20:47 2005
@@ -277,7 +277,7 @@
#ifdef CONFIG_X86_PAE
/* this gives us a page below 4GB */
- xen_contig_memory((unsigned long)pgd, 0);
+ xen_create_contiguous_region((unsigned long)pgd, 0);
#endif
if (!HAVE_SHARED_KERNEL_PMD)
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig Tue Aug 16 11:20:47 2005
@@ -329,12 +329,12 @@
# need this always enabled with GART_IOMMU for the VIA workaround
config SWIOTLB
bool
- depends on GART_IOMMU
+ depends on PCI
default y
config DUMMY_IOMMU
bool
- depends on !GART_IOMMU && !SWIOTLB
+ depends on !GART_IOMMU
default y
help
Don't use IOMMU code. This will cause problems when you have more
than 4GB
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile Tue Aug 16
10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile Tue Aug 16
11:20:47 2005
@@ -36,8 +36,9 @@
#obj-$(CONFIG_CPU_FREQ) += cpufreq/
#obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
#obj-$(CONFIG_GART_IOMMU) += pci-gart.o aperture.o
-obj-$(CONFIG_DUMMY_IOMMU) += pci-nommu.o pci-dma.o
-#obj-$(CONFIG_SWIOTLB) += swiotlb.o
+obj-$(CONFIG_DUMMY_IOMMU) += pci-nommu.o
+i386-obj-$(CONFIG_DUMMY_IOMMU) += pci-dma.o
+i386-obj-$(CONFIG_SWIOTLB) += swiotlb.o
obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_X86_PM_TIMER) += pmtimer.o
@@ -49,7 +50,7 @@
bootflag-y += ../../../i386/kernel/bootflag.o
cpuid-$(subst m,y,$(CONFIG_X86_CPUID)) += ../../../i386/kernel/cpuid.o
topology-y += ../../../i386/mach-default/topology.o
-swiotlb-$(CONFIG_SWIOTLB) += ../../../ia64/lib/swiotlb.o
+#swiotlb-$(CONFIG_SWIOTLB) += ../../../ia64/lib/swiotlb.o
microcode-$(subst m,y,$(CONFIG_MICROCODE)) += ../../../i386/kernel/microcode.o
intel_cacheinfo-y += ../../../i386/kernel/cpu/intel_cacheinfo.o
quirks-y += ../../i386/kernel/quirks.o
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c Tue Aug 16
10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c Tue Aug 16
11:20:47 2005
@@ -61,6 +61,7 @@
EXPORT_SYMBOL(dma_free_coherent);
#endif
+#if 0
int dma_supported(struct device *hwdev, u64 mask)
{
/*
@@ -76,6 +77,7 @@
return 1;
}
EXPORT_SYMBOL(dma_supported);
+#endif
int dma_get_cache_alignment(void)
{
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c Tue Aug 16 11:20:47 2005
@@ -42,10 +42,6 @@
#ifndef Dprintk
#define Dprintk(x...)
-#endif
-
-#ifdef CONFIG_GART_IOMMU
-extern int swiotlb;
#endif
extern char _stext[];
@@ -790,8 +786,6 @@
return 1;
}
-extern int swiotlb_force;
-
static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
kcore_vsyscall;
@@ -800,14 +794,9 @@
int codesize, reservedpages, datasize, initsize;
int tmp;
-#ifdef CONFIG_SWIOTLB
- if (swiotlb_force)
- swiotlb = 1;
- if (!iommu_aperture &&
- (end_pfn >= 0xffffffff>>PAGE_SHIFT || force_iommu))
- swiotlb = 1;
- if (swiotlb)
- swiotlb_init();
+#if defined(CONFIG_SWIOTLB)
+ extern void swiotlb_init(void);
+ swiotlb_init();
#endif
/* How many end-of-memory variables you have, grandma! */
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h Tue Aug
16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h Tue Aug
16 11:20:47 2005
@@ -1,11 +1,33 @@
#ifndef _ASM_I386_DMA_MAPPING_H
#define _ASM_I386_DMA_MAPPING_H
+/*
+ * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for
+ * documentation.
+ */
+
+#include <linux/config.h>
#include <linux/mm.h>
-
#include <asm/cache.h>
#include <asm/io.h>
#include <asm/scatterlist.h>
+#include <asm-i386/swiotlb.h>
+
+static inline int
+address_needs_mapping(struct device *hwdev, dma_addr_t addr)
+{
+ dma_addr_t mask = 0xffffffff;
+ /* If the device has a mask, use it, otherwise default to 32 bits */
+ if (hwdev && hwdev->dma_mask)
+ mask = *hwdev->dma_mask;
+ return (addr & ~mask) != 0;
+}
+
+static inline int
+range_straddles_page_boundary(void *p, size_t size)
+{
+ return ((((unsigned long)p & ~PAGE_MASK) + size) > PAGE_SIZE);
+}
#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h)
@@ -24,46 +46,18 @@
dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
enum dma_data_direction direction);
-static inline int
-dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
- enum dma_data_direction direction)
-{
- int i;
+extern int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, enum dma_data_direction direction);
+extern void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, enum dma_data_direction direction);
- BUG_ON(direction == DMA_NONE);
+extern dma_addr_t
+dma_map_page(struct device *dev, struct page *page, unsigned long offset,
+ size_t size, enum dma_data_direction direction);
- for (i = 0; i < nents; i++ ) {
- BUG_ON(!sg[i].page);
-
- sg[i].dma_address = page_to_phys(sg[i].page) + sg[i].offset;
- }
-
- flush_write_buffers();
- return nents;
-}
-
-static inline dma_addr_t
-dma_map_page(struct device *dev, struct page *page, unsigned long offset,
- size_t size, enum dma_data_direction direction)
-{
- BUG_ON(direction == DMA_NONE);
- return page_to_phys(page) + offset;
-}
-
-static inline void
+extern void
dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
- enum dma_data_direction direction)
-{
- BUG_ON(direction == DMA_NONE);
-}
-
-
-static inline void
-dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
- enum dma_data_direction direction)
-{
- BUG_ON(direction == DMA_NONE);
-}
+ enum dma_data_direction direction);
extern void
dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
@@ -93,34 +87,25 @@
dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
enum dma_data_direction direction)
{
+ if (swiotlb)
+ swiotlb_sync_sg_for_cpu(dev,sg,nelems,direction);
+ flush_write_buffers();
}
static inline void
dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
enum dma_data_direction direction)
{
+ if (swiotlb)
+ swiotlb_sync_sg_for_device(dev,sg,nelems,direction);
flush_write_buffers();
}
-static inline int
-dma_mapping_error(dma_addr_t dma_addr)
-{
- return 0;
-}
+extern int
+dma_mapping_error(dma_addr_t dma_addr);
-static inline int
-dma_supported(struct device *dev, u64 mask)
-{
- /*
- * we fall back to GFP_DMA when the mask isn't all 1s,
- * so we can't guarantee allocations that must be
- * within a tighter range than GFP_DMA..
- */
- if(mask < 0x00ffffff)
- return 0;
-
- return 1;
-}
+extern int
+dma_supported(struct device *dev, u64 mask);
static inline int
dma_set_mask(struct device *dev, u64 mask)
@@ -133,6 +118,7 @@
return 0;
}
+#ifdef __i386__
static inline int
dma_get_cache_alignment(void)
{
@@ -140,6 +126,9 @@
* maximum possible, to be safe */
return (1 << L1_CACHE_SHIFT_MAX);
}
+#else
+extern int dma_get_cache_alignment(void);
+#endif
#define dma_is_consistent(d) (1)
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h Tue Aug 16
10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h Tue Aug 16
11:20:47 2005
@@ -43,11 +43,8 @@
struct pci_dev;
-/* The PCI address space does equal the physical memory
- * address space. The networking and block device layers use
- * this boolean for bounce buffer decisions.
- */
-#define PCI_DMA_BUS_IS_PHYS (1)
+/* On Xen we use SWIOTLB instead of blk-specific bounce buffers. */
+#define PCI_DMA_BUS_IS_PHYS (0)
/* pci_unmap_{page,single} is a nop so... */
#define DECLARE_PCI_UNMAP_ADDR(ADDR_NAME)
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h Tue Aug
16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h Tue Aug
16 11:20:47 2005
@@ -1,89 +1,1 @@
-#ifndef _X8664_DMA_MAPPING_H
-#define _X8664_DMA_MAPPING_H 1
-
-/*
- * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for
- * documentation.
- */
-
-#include <linux/config.h>
-
-#include <asm/scatterlist.h>
-#include <asm/io.h>
-#include <asm/swiotlb.h>
-
-extern dma_addr_t bad_dma_address;
-#define dma_mapping_error(x) \
- (swiotlb ? swiotlb_dma_mapping_error(x) : ((x) == bad_dma_address))
-
-void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t
*dma_handle,
- unsigned gfp);
-void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
- dma_addr_t dma_handle);
-
-extern dma_addr_t dma_map_single(struct device *hwdev, void *ptr, size_t size,
- enum dma_data_direction direction);
-extern void dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size,
- enum dma_data_direction direction);
-
-#define dma_map_page(dev,page,offset,size,dir) \
- dma_map_single((dev), page_address(page)+(offset), (size), (dir))
-
-extern void
-dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
- enum dma_data_direction direction);
-
-extern void
-dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t
size,
- enum dma_data_direction direction);
-
-static inline void dma_sync_sg_for_cpu(struct device *hwdev,
- struct scatterlist *sg,
- int nelems, int direction)
-{
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- if (swiotlb)
- return swiotlb_sync_sg_for_cpu(hwdev,sg,nelems,direction);
-
- flush_write_buffers();
-}
-
-static inline void dma_sync_sg_for_device(struct device *hwdev,
- struct scatterlist *sg,
- int nelems, int direction)
-{
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- if (swiotlb)
- return swiotlb_sync_sg_for_device(hwdev,sg,nelems,direction);
-
- flush_write_buffers();
-}
-
-extern int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
- int nents, int direction);
-extern void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
- int nents, int direction);
-
-#define dma_unmap_page dma_unmap_single
-
-extern int dma_supported(struct device *hwdev, u64 mask);
-extern int dma_get_cache_alignment(void);
-#define dma_is_consistent(h) 1
-
-static inline int dma_set_mask(struct device *dev, u64 mask)
-{
- if (!dev->dma_mask || !dma_supported(dev, mask))
- return -EIO;
- *dev->dma_mask = mask;
- return 0;
-}
-
-static inline void dma_cache_sync(void *vaddr, size_t size, enum
dma_data_direction dir)
-{
- flush_write_buffers();
-}
-#endif
+#include <asm-i386/dma-mapping.h>
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h
--- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h Tue Aug 16
10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h Tue Aug 16
11:20:47 2005
@@ -79,7 +79,9 @@
#else
/* No IOMMU */
-#define PCI_DMA_BUS_IS_PHYS 1
+/* On Xen we use SWIOTLB instead of blk-specific bounce buffers. */
+#define PCI_DMA_BUS_IS_PHYS (0)
+
#define pci_dac_dma_supported(pci_dev, mask) 1
#define DECLARE_PCI_UNMAP_ADDR(ADDR_NAME)
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/hypervisor.h
--- a/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h Tue Aug 16 11:20:47 2005
@@ -134,7 +134,8 @@
#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
#endif /* linux < 2.6.0 */
-void xen_contig_memory(unsigned long vstart, unsigned int order);
+void xen_create_contiguous_region(unsigned long vstart, unsigned int order);
+void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order);
#ifdef CONFIG_XEN_PHYSDEV_ACCESS
/* Allocate a contiguous empty region of low memory. Return virtual start. */
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/i386/kernel/swiotlb.c
--- /dev/null Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/swiotlb.c Tue Aug 16
11:20:47 2005
@@ -0,0 +1,653 @@
+/*
+ * Dynamic DMA mapping support.
+ *
+ * This implementation is a fallback for platforms that do not support
+ * I/O TLBs (aka DMA address translation hardware).
+ * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@xxxxxxxxx>
+ * Copyright (C) 2000 Goutham Rao <goutham.rao@xxxxxxxxx>
+ * Copyright (C) 2000, 2003 Hewlett-Packard Co
+ * David Mosberger-Tang <davidm@xxxxxxxxxx>
+ * Copyright (C) 2005 Keir Fraser <keir@xxxxxxxxxxxxx>
+ */
+
+#include <linux/cache.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/bootmem.h>
+#include <linux/highmem.h>
+#include <asm/io.h>
+#include <asm/pci.h>
+#include <asm/dma.h>
+
+#define OFFSET(val,align) ((unsigned long)((val) & ( (align) - 1)))
+
+#define SG_ENT_PHYS_ADDRESS(sg) (page_to_phys((sg)->page) +
(sg)->offset)
+
+/*
+ * Maximum allowable number of contiguous slabs to map,
+ * must be a power of 2. What is the appropriate value ?
+ * The complexity of {map,unmap}_single is linearly dependent on this value.
+ */
+#define IO_TLB_SEGSIZE 128
+
+/*
+ * log of the size of each IO TLB slab. The number of slabs is command line
+ * controllable.
+ */
+#define IO_TLB_SHIFT 11
+
+int swiotlb_force;
+
+/*
+ * Used to do a quick range check in swiotlb_unmap_single and
+ * swiotlb_sync_single_*, to see if the memory was in fact allocated by this
+ * API.
+ */
+static char *io_tlb_start, *io_tlb_end;
+
+/*
+ * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
+ * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
+ */
+static unsigned long io_tlb_nslabs;
+
+/*
+ * When the IOMMU overflows we return a fallback buffer. This sets the size.
+ */
+static unsigned long io_tlb_overflow = 32*1024;
+
+void *io_tlb_overflow_buffer;
+
+/*
+ * This is a free list describing the number of free entries available from
+ * each index
+ */
+static unsigned int *io_tlb_list;
+static unsigned int io_tlb_index;
+
+/*
+ * We need to save away the original address corresponding to a mapped entry
+ * for the sync operations.
+ */
+static struct phys_addr {
+ struct page *page;
+ unsigned int offset;
+} *io_tlb_orig_addr;
+
+/*
+ * Protect the above data structures in the map and unmap calls
+ */
+static DEFINE_SPINLOCK(io_tlb_lock);
+
+static int __init
+setup_io_tlb_npages(char *str)
+{
+ if (isdigit(*str)) {
+ io_tlb_nslabs = simple_strtoul(str, &str, 0) <<
+ (PAGE_SHIFT - IO_TLB_SHIFT);
+ /* avoid tail segment of size < IO_TLB_SEGSIZE */
+ io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
+ }
+ if (*str == ',')
+ ++str;
+ /*
+ * NB. 'force' enables the swiotlb, but doesn't force its use for
+ * every DMA like it does on native Linux.
+ */
+ if (!strcmp(str, "force"))
+ swiotlb_force = 1;
+ return 1;
+}
+__setup("swiotlb=", setup_io_tlb_npages);
+/* make io_tlb_overflow tunable too? */
+
+/*
+ * Statically reserve bounce buffer space and initialize bounce buffer data
+ * structures for the software IO TLB used to implement the PCI DMA API.
+ */
+void
+swiotlb_init_with_default_size (size_t default_size)
+{
+ unsigned long i;
+
+ if (!io_tlb_nslabs) {
+ io_tlb_nslabs = (default_size >> PAGE_SHIFT);
+ io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
+ }
+
+ /*
+ * Get IO TLB memory from the low pages
+ */
+ io_tlb_start = alloc_bootmem_low_pages(io_tlb_nslabs *
+ (1 << IO_TLB_SHIFT));
+ if (!io_tlb_start)
+ panic("Cannot allocate SWIOTLB buffer");
+
+ xen_create_contiguous_region(
+ (unsigned long)io_tlb_start,
+ get_order(io_tlb_nslabs * (1 << IO_TLB_SHIFT)));
+
+ io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT);
+
+ /*
+ * Allocate and initialize the free list array. This array is used
+ * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
+ * between io_tlb_start and io_tlb_end.
+ */
+ io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
+ for (i = 0; i < io_tlb_nslabs; i++)
+ io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
+ io_tlb_index = 0;
+ io_tlb_orig_addr = alloc_bootmem(
+ io_tlb_nslabs * sizeof(*io_tlb_orig_addr));
+
+ /*
+ * Get the overflow emergency buffer
+ */
+ io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
+ printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n",
+ virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end-1));
+}
+
+void
+swiotlb_init(void)
+{
+ /* The user can forcibly enable swiotlb. */
+ if (swiotlb_force)
+ swiotlb = 1;
+
+ /*
+ * Otherwise, enable for domain 0 if the machine has 'lots of memory',
+ * which we take to mean more than 2GB.
+ */
+ if (xen_start_info.flags & SIF_INITDOMAIN) {
+ dom0_op_t op;
+ op.cmd = DOM0_PHYSINFO;
+ if ((HYPERVISOR_dom0_op(&op) == 0) &&
+ (op.u.physinfo.total_pages > 0x7ffff))
+ swiotlb = 1;
+ }
+
+ if (swiotlb)
+ swiotlb_init_with_default_size(64 * (1<<20));
+}
+
+static void
+__sync_single(struct phys_addr buffer, char *dma_addr, size_t size, int dir)
+{
+ if (PageHighMem(buffer.page)) {
+ size_t len, bytes;
+ char *dev, *host, *kmp;
+ len = size;
+ while (len != 0) {
+ if (((bytes = len) + buffer.offset) > PAGE_SIZE)
+ bytes = PAGE_SIZE - buffer.offset;
+ kmp = kmap_atomic(buffer.page, KM_SWIOTLB);
+ dev = dma_addr + size - len;
+ host = kmp + buffer.offset;
+ memcpy((dir == DMA_FROM_DEVICE) ? host : dev,
+ (dir == DMA_FROM_DEVICE) ? dev : host,
+ bytes);
+ kunmap_atomic(kmp, KM_SWIOTLB);
+ len -= bytes;
+ buffer.page++;
+ buffer.offset = 0;
+ }
+ } else {
+ char *host = (char *)phys_to_virt(
+ page_to_pseudophys(buffer.page)) + buffer.offset;
+ if (dir == DMA_FROM_DEVICE)
+ memcpy(host, dma_addr, size);
+ else if (dir == DMA_TO_DEVICE)
+ memcpy(dma_addr, host, size);
+ }
+}
+
+/*
+ * Allocates bounce buffer and returns its kernel virtual address.
+ */
+static void *
+map_single(struct device *hwdev, struct phys_addr buffer, size_t size, int dir)
+{
+ unsigned long flags;
+ char *dma_addr;
+ unsigned int nslots, stride, index, wrap;
+ int i;
+
+ /*
+ * For mappings greater than a page, we limit the stride (and
+ * hence alignment) to a page size.
+ */
+ nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
+ if (size > PAGE_SIZE)
+ stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
+ else
+ stride = 1;
+
+ BUG_ON(!nslots);
+
+ /*
+ * Find suitable number of IO TLB entries size that will fit this
+ * request and allocate a buffer from that IO TLB pool.
+ */
+ spin_lock_irqsave(&io_tlb_lock, flags);
+ {
+ wrap = index = ALIGN(io_tlb_index, stride);
+
+ if (index >= io_tlb_nslabs)
+ wrap = index = 0;
+
+ do {
+ /*
+ * If we find a slot that indicates we have 'nslots'
+ * number of contiguous buffers, we allocate the
+ * buffers from that slot and mark the entries as '0'
+ * indicating unavailable.
+ */
+ if (io_tlb_list[index] >= nslots) {
+ int count = 0;
+
+ for (i = index; i < (int)(index + nslots); i++)
+ io_tlb_list[i] = 0;
+ for (i = index - 1;
+ (OFFSET(i, IO_TLB_SEGSIZE) !=
+ IO_TLB_SEGSIZE -1) && io_tlb_list[i];
+ i--)
+ io_tlb_list[i] = ++count;
+ dma_addr = io_tlb_start +
+ (index << IO_TLB_SHIFT);
+
+ /*
+ * Update the indices to avoid searching in
+ * the next round.
+ */
+ io_tlb_index =
+ ((index + nslots) < io_tlb_nslabs
+ ? (index + nslots) : 0);
+
+ goto found;
+ }
+ index += stride;
+ if (index >= io_tlb_nslabs)
+ index = 0;
+ } while (index != wrap);
+
+ spin_unlock_irqrestore(&io_tlb_lock, flags);
+ return NULL;
+ }
+ found:
+ spin_unlock_irqrestore(&io_tlb_lock, flags);
+
+ /*
+ * Save away the mapping from the original address to the DMA address.
+ * This is needed when we sync the memory. Then we sync the buffer if
+ * needed.
+ */
+ io_tlb_orig_addr[index] = buffer;
+ if ((dir == DMA_TO_DEVICE) || (dir == DMA_BIDIRECTIONAL))
+ __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
+
+ return dma_addr;
+}
+
+/*
+ * dma_addr is the kernel virtual address of the bounce buffer to unmap.
+ */
+static void
+unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
+{
+ unsigned long flags;
+ int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
+ int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
+ struct phys_addr buffer = io_tlb_orig_addr[index];
+
+ /*
+ * First, sync the memory before unmapping the entry
+ */
+ if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))
+ __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
+
+ /*
+ * Return the buffer to the free list by setting the corresponding
+ * entries to indicate the number of contigous entries available.
+ * While returning the entries to the free list, we merge the entries
+ * with slots below and above the pool being returned.
+ */
+ spin_lock_irqsave(&io_tlb_lock, flags);
+ {
+ count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
+ io_tlb_list[index + nslots] : 0);
+ /*
+ * Step 1: return the slots to the free list, merging the
+ * slots with superceeding slots
+ */
+ for (i = index + nslots - 1; i >= index; i--)
+ io_tlb_list[i] = ++count;
+ /*
+ * Step 2: merge the returned slots with the preceding slots,
+ * if available (non zero)
+ */
+ for (i = index - 1;
+ (OFFSET(i, IO_TLB_SEGSIZE) !=
+ IO_TLB_SEGSIZE -1) && io_tlb_list[i];
+ i--)
+ io_tlb_list[i] = ++count;
+ }
+ spin_unlock_irqrestore(&io_tlb_lock, flags);
+}
+
+static void
+sync_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
+{
+ int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
+ struct phys_addr buffer = io_tlb_orig_addr[index];
+ BUG_ON((dir != DMA_FROM_DEVICE) && (dir != DMA_TO_DEVICE));
+ __sync_single(buffer, dma_addr, size, dir);
+}
+
+static void
+swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
+{
+ /*
+ * Ran out of IOMMU space for this operation. This is very bad.
+ * Unfortunately the drivers cannot handle this operation properly.
+ * unless they check for pci_dma_mapping_error (most don't)
+ * When the mapping is small enough return a static buffer to limit
+ * the damage, or panic when the transfer is too big.
+ */
+ printk(KERN_ERR "PCI-DMA: Out of SW-IOMMU space for %lu bytes at "
+ "device %s\n", (unsigned long)size, dev ? dev->bus_id : "?");
+
+ if (size > io_tlb_overflow && do_panic) {
+ if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
+ panic("PCI-DMA: Memory would be corrupted\n");
+ if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL)
+ panic("PCI-DMA: Random memory would be DMAed\n");
+ }
+}
+
+/*
+ * Map a single buffer of the indicated size for DMA in streaming mode. The
+ * PCI address to use is returned.
+ *
+ * Once the device is given the dma address, the device owns this memory until
+ * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
+ */
+dma_addr_t
+swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
+{
+ dma_addr_t dev_addr = virt_to_bus(ptr);
+ void *map;
+ struct phys_addr buffer;
+
+ BUG_ON(dir == DMA_NONE);
+
+ /*
+ * If the pointer passed in happens to be in the device's DMA window,
+ * we can safely return the device addr and not worry about bounce
+ * buffering it.
+ */
+ if (!range_straddles_page_boundary(ptr, size) &&
+ !address_needs_mapping(hwdev, dev_addr))
+ return dev_addr;
+
+ /*
+ * Oh well, have to allocate and map a bounce buffer.
+ */
+ buffer.page = virt_to_page(ptr);
+ buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
+ map = map_single(hwdev, buffer, size, dir);
+ if (!map) {
+ swiotlb_full(hwdev, size, dir, 1);
+ map = io_tlb_overflow_buffer;
+ }
+
+ dev_addr = virt_to_bus(map);
+
+ /*
+ * Ensure that the address returned is DMA'ble
+ */
+ if (address_needs_mapping(hwdev, dev_addr))
+ panic("map_single: bounce buffer is not DMA'ble");
+
+ return dev_addr;
+}
+
+/*
+ * Unmap a single streaming mode DMA translation. The dma_addr and size must
+ * match what was provided for in a previous swiotlb_map_single call. All
+ * other usages are undefined.
+ *
+ * After this call, reads by the cpu to the buffer are guaranteed to see
+ * whatever the device wrote there.
+ */
+void
+swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
+ int dir)
+{
+ char *dma_addr = bus_to_virt(dev_addr);
+
+ BUG_ON(dir == DMA_NONE);
+ if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+ unmap_single(hwdev, dma_addr, size, dir);
+}
+
+/*
+ * Make physical memory consistent for a single streaming mode DMA translation
+ * after a transfer.
+ *
+ * If you perform a swiotlb_map_single() but wish to interrogate the buffer
+ * using the cpu, yet do not wish to teardown the PCI dma mapping, you must
+ * call this function before doing so. At the next point you give the PCI dma
+ * address back to the card, you must first perform a
+ * swiotlb_dma_sync_for_device, and then the device again owns the buffer
+ */
+void
+swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
+ size_t size, int dir)
+{
+ char *dma_addr = bus_to_virt(dev_addr);
+
+ BUG_ON(dir == DMA_NONE);
+ if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+ sync_single(hwdev, dma_addr, size, dir);
+}
+
+void
+swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
+ size_t size, int dir)
+{
+ char *dma_addr = bus_to_virt(dev_addr);
+
+ BUG_ON(dir == DMA_NONE);
+ if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+ sync_single(hwdev, dma_addr, size, dir);
+}
+
+/*
+ * Map a set of buffers described by scatterlist in streaming mode for DMA.
+ * This is the scatter-gather version of the above swiotlb_map_single
+ * interface. Here the scatter gather list elements are each tagged with the
+ * appropriate dma address and length. They are obtained via
+ * sg_dma_{address,length}(SG).
+ *
+ * NOTE: An implementation may be able to use a smaller number of
+ * DMA address/length pairs than there are SG table elements.
+ * (for example via virtual mapping capabilities)
+ * The routine returns the number of addr/length pairs actually
+ * used, at most nents.
+ *
+ * Device ownership issues as mentioned above for swiotlb_map_single are the
+ * same here.
+ */
+int
+swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
+ int dir)
+{
+ struct phys_addr buffer;
+ dma_addr_t dev_addr;
+ char *map;
+ int i;
+
+ BUG_ON(dir == DMA_NONE);
+
+ for (i = 0; i < nelems; i++, sg++) {
+ dev_addr = SG_ENT_PHYS_ADDRESS(sg);
+ if (address_needs_mapping(hwdev, dev_addr)) {
+ buffer.page = sg->page;
+ buffer.offset = sg->offset;
+ map = map_single(hwdev, buffer, sg->length, dir);
+ if (!map) {
+ /* Don't panic here, we expect map_sg users
+ to do proper error handling. */
+ swiotlb_full(hwdev, sg->length, dir, 0);
+ swiotlb_unmap_sg(hwdev, sg - i, i, dir);
+ sg[0].dma_length = 0;
+ return 0;
+ }
+ sg->dma_address = (dma_addr_t)virt_to_bus(map);
+ } else
+ sg->dma_address = dev_addr;
+ sg->dma_length = sg->length;
+ }
+ return nelems;
+}
+
+/*
+ * Unmap a set of streaming mode DMA translations. Again, cpu read rules
+ * concerning calls here are the same as for swiotlb_unmap_single() above.
+ */
+void
+swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
+ int dir)
+{
+ int i;
+
+ BUG_ON(dir == DMA_NONE);
+
+ for (i = 0; i < nelems; i++, sg++)
+ if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+ unmap_single(hwdev,
+ (void *)bus_to_virt(sg->dma_address),
+ sg->dma_length, dir);
+}
+
+/*
+ * Make physical memory consistent for a set of streaming mode DMA translations
+ * after a transfer.
+ *
+ * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
+ * and usage.
+ */
+void
+swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int dir)
+{
+ int i;
+
+ BUG_ON(dir == DMA_NONE);
+
+ for (i = 0; i < nelems; i++, sg++)
+ if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+ sync_single(hwdev,
+ (void *)bus_to_virt(sg->dma_address),
+ sg->dma_length, dir);
+}
+
+void
+swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
+ int nelems, int dir)
+{
+ int i;
+
+ BUG_ON(dir == DMA_NONE);
+
+ for (i = 0; i < nelems; i++, sg++)
+ if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
+ sync_single(hwdev,
+ (void *)bus_to_virt(sg->dma_address),
+ sg->dma_length, dir);
+}
+
+dma_addr_t
+swiotlb_map_page(struct device *hwdev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction direction)
+{
+ struct phys_addr buffer;
+ dma_addr_t dev_addr;
+ char *map;
+
+ dev_addr = page_to_phys(page) + offset;
+ if (address_needs_mapping(hwdev, dev_addr)) {
+ buffer.page = page;
+ buffer.offset = offset;
+ map = map_single(hwdev, buffer, size, direction);
+ if (!map) {
+ swiotlb_full(hwdev, size, direction, 1);
+ map = io_tlb_overflow_buffer;
+ }
+ dev_addr = (dma_addr_t)virt_to_bus(map);
+ }
+
+ return dev_addr;
+}
+
+void
+swiotlb_unmap_page(struct device *hwdev, dma_addr_t dma_address,
+ size_t size, enum dma_data_direction direction)
+{
+ char *dma_addr = bus_to_virt(dma_address);
+
+ BUG_ON(direction == DMA_NONE);
+ if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
+ unmap_single(hwdev, dma_addr, size, direction);
+}
+
+int
+swiotlb_dma_mapping_error(dma_addr_t dma_addr)
+{
+ return (dma_addr == virt_to_bus(io_tlb_overflow_buffer));
+}
+
+/*
+ * Return whether the given PCI device DMA address mask can be supported
+ * properly. For example, if your device can only drive the low 24-bits
+ * during PCI bus mastering, then you would pass 0x00ffffff as the mask to
+ * this function.
+ */
+int
+swiotlb_dma_supported (struct device *hwdev, u64 mask)
+{
+ return (mask >= 0xffffffffUL);
+}
+
+EXPORT_SYMBOL(swiotlb_init);
+EXPORT_SYMBOL(swiotlb_map_single);
+EXPORT_SYMBOL(swiotlb_unmap_single);
+EXPORT_SYMBOL(swiotlb_map_sg);
+EXPORT_SYMBOL(swiotlb_unmap_sg);
+EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
+EXPORT_SYMBOL(swiotlb_sync_single_for_device);
+EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
+EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
+EXPORT_SYMBOL(swiotlb_map_page);
+EXPORT_SYMBOL(swiotlb_unmap_page);
+EXPORT_SYMBOL(swiotlb_dma_mapping_error);
+EXPORT_SYMBOL(swiotlb_dma_supported);
+
+/*
+ * Local variables:
+ * c-file-style: "linux"
+ * indent-tabs-mode: t
+ * c-indent-level: 8
+ * c-basic-offset: 8
+ * tab-width: 8
+ * End:
+ */
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/asm-i386/kmap_types.h
--- /dev/null Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/kmap_types.h Tue Aug
16 11:20:47 2005
@@ -0,0 +1,32 @@
+#ifndef _ASM_KMAP_TYPES_H
+#define _ASM_KMAP_TYPES_H
+
+#include <linux/config.h>
+
+#ifdef CONFIG_DEBUG_HIGHMEM
+# define D(n) __KM_FENCE_##n ,
+#else
+# define D(n)
+#endif
+
+enum km_type {
+D(0) KM_BOUNCE_READ,
+D(1) KM_SKB_SUNRPC_DATA,
+D(2) KM_SKB_DATA_SOFTIRQ,
+D(3) KM_USER0,
+D(4) KM_USER1,
+D(5) KM_BIO_SRC_IRQ,
+D(6) KM_BIO_DST_IRQ,
+D(7) KM_PTE0,
+D(8) KM_PTE1,
+D(9) KM_IRQ0,
+D(10) KM_IRQ1,
+D(11) KM_SOFTIRQ0,
+D(12) KM_SOFTIRQ1,
+D(13) KM_SWIOTLB,
+D(14) KM_TYPE_NR
+};
+
+#undef D
+
+#endif
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/asm-i386/scatterlist.h
--- /dev/null Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/scatterlist.h Tue Aug
16 11:20:47 2005
@@ -0,0 +1,22 @@
+#ifndef _I386_SCATTERLIST_H
+#define _I386_SCATTERLIST_H
+
+struct scatterlist {
+ struct page *page;
+ unsigned int offset;
+ unsigned int length;
+ dma_addr_t dma_address;
+ unsigned int dma_length;
+};
+
+/* These macros should be used after a pci_map_sg call has been done
+ * to get bus addresses of each of the SG entries and their lengths.
+ * You should only work with the number of sg entries pci_map_sg
+ * returns.
+ */
+#define sg_dma_address(sg) ((sg)->dma_address)
+#define sg_dma_len(sg) ((sg)->dma_length)
+
+#define ISA_DMA_THRESHOLD (0x00ffffff)
+
+#endif /* !(_I386_SCATTERLIST_H) */
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/include/asm-xen/asm-i386/swiotlb.h
--- /dev/null Tue Aug 16 10:12:18 2005
+++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/swiotlb.h Tue Aug 16
11:20:47 2005
@@ -0,0 +1,42 @@
+#ifndef _ASM_SWIOTLB_H
+#define _ASM_SWIOTLB_H 1
+
+#include <linux/config.h>
+
+/* SWIOTLB interface */
+
+extern dma_addr_t swiotlb_map_single(struct device *hwdev, void *ptr, size_t
size,
+ int dir);
+extern void swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr,
+ size_t size, int dir);
+extern void swiotlb_sync_single_for_cpu(struct device *hwdev,
+ dma_addr_t dev_addr,
+ size_t size, int dir);
+extern void swiotlb_sync_single_for_device(struct device *hwdev,
+ dma_addr_t dev_addr,
+ size_t size, int dir);
+extern void swiotlb_sync_sg_for_cpu(struct device *hwdev,
+ struct scatterlist *sg, int nelems,
+ int dir);
+extern void swiotlb_sync_sg_for_device(struct device *hwdev,
+ struct scatterlist *sg, int nelems,
+ int dir);
+extern int swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction);
+extern void swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg,
+ int nents, int direction);
+extern int swiotlb_dma_mapping_error(dma_addr_t dma_addr);
+extern dma_addr_t swiotlb_map_page(struct device *hwdev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction direction);
+extern void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dma_address,
+ size_t size, enum dma_data_direction direction);
+extern int swiotlb_dma_supported(struct device *hwdev, u64 mask);
+
+#ifdef CONFIG_SWIOTLB
+extern int swiotlb;
+#else
+#define swiotlb 0
+#endif
+
+#endif
diff -r eaf498f1ffde -r 3d187585c141
linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c
--- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c Tue Aug 16
10:12:18 2005
+++ /dev/null Tue Aug 16 11:20:47 2005
@@ -1,336 +0,0 @@
-/*
- * Dynamic DMA mapping support.
- */
-
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/pci.h>
-#include <linux/module.h>
-#include <asm/io.h>
-#include <asm-xen/balloon.h>
-
-/* Map a set of buffers described by scatterlist in streaming
- * mode for DMA. This is the scatter-gather version of the
- * above pci_map_single interface. Here the scatter gather list
- * elements are each tagged with the appropriate dma address
- * and length. They are obtained via sg_dma_{address,length}(SG).
- *
- * NOTE: An implementation may be able to use a smaller number of
- * DMA address/length pairs than there are SG table elements.
- * (for example via virtual mapping capabilities)
- * The routine returns the number of addr/length pairs actually
- * used, at most nents.
- *
- * Device ownership issues as mentioned above for pci_map_single are
- * the same here.
- */
-int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
- int nents, int direction)
-{
- int i;
-
- BUG_ON(direction == DMA_NONE);
- for (i = 0; i < nents; i++ ) {
- struct scatterlist *s = &sg[i];
- BUG_ON(!s->page);
- s->dma_address = virt_to_bus(page_address(s->page) +s->offset);
- s->dma_length = s->length;
- }
- return nents;
-}
-
-EXPORT_SYMBOL(dma_map_sg);
-
-/* Unmap a set of streaming mode DMA translations.
- * Again, cpu read rules concerning calls here are the same as for
- * pci_unmap_single() above.
- */
-void dma_unmap_sg(struct device *dev, struct scatterlist *sg,
- int nents, int dir)
-{
- int i;
- for (i = 0; i < nents; i++) {
- struct scatterlist *s = &sg[i];
- BUG_ON(s->page == NULL);
- BUG_ON(s->dma_address == 0);
- dma_unmap_single(dev, s->dma_address, s->dma_length, dir);
- }
-}
-
-EXPORT_SYMBOL(dma_unmap_sg);
-
-struct dma_coherent_mem {
- void *virt_base;
- u32 device_base;
- int size;
- int flags;
- unsigned long *bitmap;
-};
-
-void *dma_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t *dma_handle, unsigned gfp)
-{
- void *ret;
- unsigned int order = get_order(size);
- unsigned long vstart;
-
- struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
-
- /* ignore region specifiers */
- gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
-
- if (mem) {
- int page = bitmap_find_free_region(mem->bitmap, mem->size,
- order);
- if (page >= 0) {
- *dma_handle = mem->device_base + (page << PAGE_SHIFT);
- ret = mem->virt_base + (page << PAGE_SHIFT);
- memset(ret, 0, size);
- return ret;
- }
- if (mem->flags & DMA_MEMORY_EXCLUSIVE)
- return NULL;
- }
-
- if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
- gfp |= GFP_DMA;
-
- vstart = __get_free_pages(gfp, order);
- ret = (void *)vstart;
- if (ret == NULL)
- return ret;
-
- xen_contig_memory(vstart, order);
-
- memset(ret, 0, size);
- *dma_handle = virt_to_bus(ret);
-
- return ret;
-}
-EXPORT_SYMBOL(dma_alloc_coherent);
-
-void dma_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_handle)
-{
- struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
- int order = get_order(size);
-
- if (mem && vaddr >= mem->virt_base && vaddr < (mem->virt_base +
(mem->size << PAGE_SHIFT))) {
- int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
-
- bitmap_release_region(mem->bitmap, page, order);
- } else
- free_pages((unsigned long)vaddr, order);
-}
-EXPORT_SYMBOL(dma_free_coherent);
-
-#if 0
-int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
- dma_addr_t device_addr, size_t size, int flags)
-{
- void __iomem *mem_base;
- int pages = size >> PAGE_SHIFT;
- int bitmap_size = (pages + 31)/32;
-
- if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
- goto out;
- if (!size)
- goto out;
- if (dev->dma_mem)
- goto out;
-
- /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
-
- mem_base = ioremap(bus_addr, size);
- if (!mem_base)
- goto out;
-
- dev->dma_mem = kmalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
- if (!dev->dma_mem)
- goto out;
- memset(dev->dma_mem, 0, sizeof(struct dma_coherent_mem));
- dev->dma_mem->bitmap = kmalloc(bitmap_size, GFP_KERNEL);
- if (!dev->dma_mem->bitmap)
- goto free1_out;
- memset(dev->dma_mem->bitmap, 0, bitmap_size);
-
- dev->dma_mem->virt_base = mem_base;
- dev->dma_mem->device_base = device_addr;
- dev->dma_mem->size = pages;
- dev->dma_mem->flags = flags;
-
- if (flags & DMA_MEMORY_MAP)
- return DMA_MEMORY_MAP;
-
- return DMA_MEMORY_IO;
-
- free1_out:
- kfree(dev->dma_mem->bitmap);
- out:
- return 0;
-}
-EXPORT_SYMBOL(dma_declare_coherent_memory);
-
-void dma_release_declared_memory(struct device *dev)
-{
- struct dma_coherent_mem *mem = dev->dma_mem;
-
- if(!mem)
- return;
- dev->dma_mem = NULL;
- iounmap(mem->virt_base);
- kfree(mem->bitmap);
- kfree(mem);
-}
-EXPORT_SYMBOL(dma_release_declared_memory);
-
-void *dma_mark_declared_memory_occupied(struct device *dev,
- dma_addr_t device_addr, size_t size)
-{
- struct dma_coherent_mem *mem = dev->dma_mem;
- int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >>
PAGE_SHIFT;
- int pos, err;
-
- if (!mem)
- return ERR_PTR(-EINVAL);
-
- pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
- err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
- if (err != 0)
- return ERR_PTR(err);
- return mem->virt_base + (pos << PAGE_SHIFT);
-}
-EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
-#endif
-
-static LIST_HEAD(dma_map_head);
-static DEFINE_SPINLOCK(dma_map_lock);
-struct dma_map_entry {
- struct list_head list;
- dma_addr_t dma;
- char *bounce, *host;
- size_t size;
-};
-#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d)))
-
-dma_addr_t
-dma_map_single(struct device *dev, void *ptr, size_t size,
- enum dma_data_direction direction)
-{
- struct dma_map_entry *ent;
- void *bnc;
- dma_addr_t dma;
- unsigned long flags;
-
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- /*
- * Even if size is sub-page, the buffer may still straddle a page
- * boundary. Take into account buffer start offset. All other calls are
- * conservative and always search the dma_map list if it's non-empty.
- */
- if (((((unsigned long)ptr) & ~PAGE_MASK) + size) <= PAGE_SIZE) {
- dma = virt_to_bus(ptr);
- } else {
- BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, GFP_ATOMIC))
== NULL);
- BUG_ON((ent = kmalloc(sizeof(*ent), GFP_ATOMIC)) == NULL);
- if (direction != DMA_FROM_DEVICE)
- memcpy(bnc, ptr, size);
- ent->dma = dma;
- ent->bounce = bnc;
- ent->host = ptr;
- ent->size = size;
- spin_lock_irqsave(&dma_map_lock, flags);
- list_add(&ent->list, &dma_map_head);
- spin_unlock_irqrestore(&dma_map_lock, flags);
- }
-
- if ((dma+size) & ~*dev->dma_mask)
- out_of_line_bug();
- return dma;
-}
-EXPORT_SYMBOL(dma_map_single);
-
-void
-dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
- enum dma_data_direction direction)
-{
- struct dma_map_entry *ent;
- unsigned long flags;
-
- if (direction == DMA_NONE)
- out_of_line_bug();
-
- /* Fast-path check: are there any multi-page DMA mappings? */
- if (!list_empty(&dma_map_head)) {
- spin_lock_irqsave(&dma_map_lock, flags);
- list_for_each_entry ( ent, &dma_map_head, list ) {
- if (DMA_MAP_MATCHES(ent, dma_addr)) {
- list_del(&ent->list);
- break;
- }
- }
- spin_unlock_irqrestore(&dma_map_lock, flags);
- if (&ent->list != &dma_map_head) {
- BUG_ON(dma_addr != ent->dma);
- BUG_ON(size != ent->size);
- if (direction != DMA_TO_DEVICE)
- memcpy(ent->host, ent->bounce, size);
- dma_free_coherent(dev, size, ent->bounce, ent->dma);
- kfree(ent);
- }
- }
-}
-EXPORT_SYMBOL(dma_unmap_single);
-
-void
-dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
- enum dma_data_direction direction)
-{
- struct dma_map_entry *ent;
- unsigned long flags, off;
-
- /* Fast-path check: are there any multi-page DMA mappings? */
- if (!list_empty(&dma_map_head)) {
- spin_lock_irqsave(&dma_map_lock, flags);
- list_for_each_entry ( ent, &dma_map_head, list )
- if (DMA_MAP_MATCHES(ent, dma_handle))
- break;
- spin_unlock_irqrestore(&dma_map_lock, flags);
- if (&ent->list != &dma_map_head) {
- off = dma_handle - ent->dma;
- BUG_ON((off + size) > ent->size);
- /*if (direction != DMA_TO_DEVICE)*/
- memcpy(ent->host+off, ent->bounce+off, size);
- }
- }
-}
-EXPORT_SYMBOL(dma_sync_single_for_cpu);
-
-void
-dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t
size,
- enum dma_data_direction direction)
-{
- struct dma_map_entry *ent;
- unsigned long flags, off;
-
- /* Fast-path check: are there any multi-page DMA mappings? */
- if (!list_empty(&dma_map_head)) {
- spin_lock_irqsave(&dma_map_lock, flags);
- list_for_each_entry ( ent, &dma_map_head, list )
- if (DMA_MAP_MATCHES(ent, dma_handle))
- break;
- spin_unlock_irqrestore(&dma_map_lock, flags);
- if (&ent->list != &dma_map_head) {
- off = dma_handle - ent->dma;
- BUG_ON((off + size) > ent->size);
- /*if (direction != DMA_FROM_DEVICE)*/
- memcpy(ent->bounce+off, ent->host+off, size);
- }
- }
-
- flush_write_buffers();
-}
-EXPORT_SYMBOL(dma_sync_single_for_device);
_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog
|
Home