WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-devel

[Xen-devel] [RFC PATCH 1/2] Xen Calgary IOMMU support - Xen bits

To: xen-devel <xen-devel@xxxxxxxxxxxxxxxxxxx>
Subject: [Xen-devel] [RFC PATCH 1/2] Xen Calgary IOMMU support - Xen bits
From: Muli Ben-Yehuda <mulix@xxxxxxxxx>
Date: Sun, 26 Mar 2006 23:10:48 +0200
Cc: muli@xxxxxxxxxx, Jimi Xenidis <jimix@xxxxxxxxxxxxxx>, amar@xxxxxxxxxx, asit.k.mallick@xxxxxxxxx, leendert@xxxxxxxxxx, okrieg@xxxxxxxxxx, elsie.wahlig@xxxxxxx, jun.nakajima@xxxxxxxxx, Jon Mason <jdmason@xxxxxxxxxx>, taufikm@xxxxxxxxxx
Delivery-date: Sun, 26 Mar 2006 21:12:52 +0000
Envelope-to: www-data@xxxxxxxxxxxxxxxxxxx
In-reply-to: <20060326210809.GA4869@xxxxxxxxxxxxxxxxxxx>
List-help: <mailto:xen-devel-request@lists.xensource.com?subject=help>
List-id: Xen developer discussion <xen-devel.lists.xensource.com>
List-post: <mailto:xen-devel@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=unsubscribe>
References: <20060326210809.GA4869@xxxxxxxxxxxxxxxxxxx>
Sender: xen-devel-bounces@xxxxxxxxxxxxxxxxxxx
User-agent: Mutt/1.5.11+cvs20060126
On Sun, Mar 26, 2006 at 11:08:09PM +0200, Muli Ben-Yehuda wrote:

> The following two patches, against the xen-unstable and linux-2.6-xen
> repositories, introduce proof-of-concept IOMMU support in Xen.

Signed-off-by: Muli Ben-Yehuda <mulix@xxxxxxxxx>
Signed-off-by: Jon Mason <jdmason@xxxxxxxxxx>

diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/arch/x86/x86_64/entry.S 
xen/xen/arch/x86/x86_64/entry.S
--- vanilla.xen/xen/arch/x86/x86_64/entry.S     2006-03-16 09:53:07.000000000 
+0200
+++ xen/xen/arch/x86/x86_64/entry.S     2006-03-20 15:15:00.000000000 +0200
@@ -557,6 +557,8 @@ ENTRY(hypercall_table)
         .quad do_acm_op
         .quad do_nmi_op
         .quad do_arch_sched_op_new
+        .quad do_iommu_map          /* 30 */
+        .quad do_iommu_unmap
         .rept NR_hypercalls-((.-hypercall_table)/8)
         .quad do_ni_hypercall
         .endr
@@ -592,6 +594,8 @@ ENTRY(hypercall_args_table)
         .byte 1 /* do_acm_op            */
         .byte 2 /* do_nmi_op            */
         .byte 2 /* do_arch_sched_op_new */
+        .byte 5 /* do_iommu_map         */  /* 30 */
+        .byte 3 /* do_iommu_unmap       */
         .rept NR_hypercalls-(.-hypercall_args_table)
         .byte 0 /* do_ni_hypercall      */
         .endr
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/common/calgary.c 
xen/xen/common/calgary.c
--- vanilla.xen/xen/common/calgary.c    1970-01-01 02:00:00.000000000 +0200
+++ xen/xen/common/calgary.c    2006-03-26 21:57:45.000000000 +0200
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2006 Muli Ben-Yehuda <mulix@xxxxxxxxx>, IBM Corporation
+ */
+
+#include <xen/lib.h>
+#include <xen/iommu.h>
+#include <xen/calgary.h>
+#include <xen/errno.h>
+#include <xen/mm.h>
+#include <asm/types.h>
+#include <asm/system.h>
+
+#define swab64(x) \
+({ \
+    u64 __x = (x); \
+    ((u64)( \
+        (u64)(((u64)(__x) & (u64)0x00000000000000ffULL) << 56) | \
+        (u64)(((u64)(__x) & (u64)0x000000000000ff00ULL) << 40) | \
+        (u64)(((u64)(__x) & (u64)0x0000000000ff0000ULL) << 24) | \
+        (u64)(((u64)(__x) & (u64)0x00000000ff000000ULL) <<  8) | \
+        (u64)(((u64)(__x) & (u64)0x000000ff00000000ULL) >>  8) | \
+        (u64)(((u64)(__x) & (u64)0x0000ff0000000000ULL) >> 24) | \
+        (u64)(((u64)(__x) & (u64)0x00ff000000000000ULL) >> 40) | \
+        (u64)(((u64)(__x) & (u64)0xff00000000000000ULL) >> 56) )); \
+})
+
+/* flush a tce at 'tceaddr' to main memory */
+static inline void flush_tce(void* tceaddr)
+{
+       /* a single tce can't cross a cache line */
+       if (cpu_has_clflush)
+               asm volatile("clflush (%0)" :: "r" (tceaddr));
+       else
+               asm volatile("wbinvd":::"memory");
+}
+
+static void tce_build(struct iommu_table* tbl, u64 index, u64 mfn, u32 access)
+{
+    u64* tp;
+    u64 t;
+
+    BUG_ON(!tbl);
+
+    t = (mfn << TCE_RPN_SHIFT) | access;
+
+    tp = ((u64*)tbl->ptr) + index;
+    *tp = swab64(t);
+
+    /* make sure HW sees it */
+    flush_tce(tp);
+}
+
+static inline u64 iospace_size_to_num_table_entries(u64 maxaddr)
+{
+    u64 ret;
+    u64 order;
+
+    order = get_order_from_bytes(maxaddr >> 13);
+
+    if (order > TCE_MAX_TABLE_SIZE)
+        order = TCE_MAX_TABLE_SIZE;
+
+    ret = (1 << order) * 0x2000;
+
+    return ret;
+}
+
+int calgary_create_io_space(u64 size, u32 bdf, void** space, u64* rootmfn)
+{
+    struct iommu_table* tbl;
+    int ret;
+    u64 bytes;
+
+    tbl = xmalloc(struct iommu_table);
+    if (!tbl) {
+        ret = -ENOMEM;
+        goto done;
+    }
+
+    tbl->size = iospace_size_to_num_table_entries(size); /* number of entries 
*/
+    bytes = tbl->size * TCE_ENTRY_SIZE;
+
+    tbl->ptr = alloc_xenheap_pages(get_order_from_bytes(bytes));
+    if (!tbl->ptr) {
+        ret = -ENODEV;
+        goto free_table;
+    }
+
+    *space = tbl;
+    *rootmfn = virt_to_maddr(tbl->ptr) >> PAGE_SHIFT;
+    return 0;
+
+free_table:
+    xfree(tbl);
+    *space = NULL;
+    *rootmfn = 0;
+done:
+    BUG_ON(ret);
+    return ret;
+}
+
+void calgary_destroy_io_space(void* space)
+{
+    struct iommu_table* tbl = space;
+    u64 bytes = tbl->size * TCE_ENTRY_SIZE;
+
+    free_xenheap_pages(tbl->ptr, get_order_from_bytes(bytes));
+    tbl->ptr = NULL;
+}
+
+u64 calgary_map(void* space, u64 ioaddr, u64 mfn, u32 access, u32 bdf, u32 
size)
+{
+    struct iommu_table* tbl = space;
+    u64 index = ioaddr >> PAGE_SHIFT;
+
+    BUG_ON(size != PAGE_SIZE);
+
+    tce_build(tbl, index, mfn, access);
+
+    return ioaddr;
+}
+
+int calgary_unmap(void* space, u64 ioaddr, u32 bdf, u32 size)
+{
+    struct iommu_table* tbl = space;
+    u64 index = ioaddr >> PAGE_SHIFT;
+
+    BUG_ON(size != PAGE_SIZE);
+
+    /* set the tce to all 0's */
+    tce_build(tbl, index, 0, 0);
+
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/common/dom0_ops.c 
xen/xen/common/dom0_ops.c
--- vanilla.xen/xen/common/dom0_ops.c   2006-03-05 13:44:48.000000000 +0200
+++ xen/xen/common/dom0_ops.c   2006-03-07 15:28:26.000000000 +0200
@@ -18,6 +18,7 @@
 #include <xen/console.h>
 #include <xen/iocap.h>
 #include <xen/guest_access.h>
+#include <xen/iommu.h>
 #include <asm/current.h>
 #include <public/dom0_ops.h>
 #include <public/sched_ctl.h>
@@ -682,6 +683,20 @@ long do_dom0_op(GUEST_HANDLE(dom0_op_t) 
     break;
 #endif
 
+    case DOM0_IOMMU_CREATE_IO_SPACE:
+    {
+        ret = iommu_create_io_space(u_dom0_op, op);
+    }
+    break;
+
+    case DOM0_IOMMU_DESTROY_IO_SPACE:
+    {
+        u32 bdf  = op->u.iommu_destroy_io_space.bdf;
+        
+        ret = iommu_destroy_io_space(bdf);
+    }
+    break;
+
     default:
         ret = arch_do_dom0_op(op, u_dom0_op);
         break;
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/common/iommu.c 
xen/xen/common/iommu.c
--- vanilla.xen/xen/common/iommu.c      1970-01-01 02:00:00.000000000 +0200
+++ xen/xen/common/iommu.c      2006-03-26 15:15:12.000000000 +0200
@@ -0,0 +1,111 @@
+/*
+ * Copyright (C) 2006 Muli Ben-Yehuda <mulix@xxxxxxxxx>, IBM Corporation
+ */
+
+#include <xen/iommu.h>
+#include <xen/calgary.h>
+#include <xen/guest_access.h>
+
+/*
+ * TODOs:
+ * - use a tree sorted by BDF to hold IO spaces
+ * - add cache flushing interfaces - all entries, single entry, range of 
entries
+ * - TODO: add iommu_ops with dynamic registration, call iommu_ops->op for ops
+ */
+
+#define NUM_IOMMU_SPACES 2
+void* iommu_space[NUM_IOMMU_SPACES];
+
+int iommu_create_io_space(GUEST_HANDLE(dom0_op_t) u_dom0_op, struct dom0_op* 
op)
+{
+    void* space;
+    int ret;
+    u64 size = op->u.iommu_create_io_space.size;
+    u32 bdf = op->u.iommu_create_io_space.bdf;
+    u64* rootmfn = &op->u.iommu_create_io_space.rootmfn;
+
+    printk("%s called (size %"PRIu64" bdf 0x%x)!\n", __func__,
+           size, bdf);
+
+    ret = calgary_create_io_space(size, bdf, &space, rootmfn);
+    if (ret)
+        goto done;
+
+    printk("space %p rootmfn %"PRIx64"\n", space, *rootmfn);
+    if (bdf == 0)
+        iommu_space[0] = space;
+    else
+        iommu_space[1] = space;
+
+    ret = 0;
+
+    /* FIXME: cleanup if fails */
+    if ( copy_to_guest(u_dom0_op, op, 1) )
+        ret = -EFAULT;
+done:
+    return ret;
+}
+
+/* should this take BDF instead? */
+int iommu_destroy_io_space(u32 bdf)
+{
+    void** space;
+
+    if (bdf == 0)
+        space = &iommu_space[0];
+    else
+        space = &iommu_space[1];
+
+    calgary_destroy_io_space(*space);
+    *space = NULL;
+
+    return 0;
+}
+
+static inline void* find_io_space(u32 bdf)
+{
+    void* space;
+
+    space = (bdf == 0 ? iommu_space[0] : iommu_space[1]);
+
+    return space;
+}
+
+/* map takes a single "entry" - use hypercalls for batching */
+u64 do_iommu_map(u64 ioaddr, u64 mfn, u32 access, u32 bdf, u32 size)
+{
+    void* space;
+    u64 retaddr;
+
+    space = find_io_space(bdf);
+
+    retaddr = calgary_map(space, ioaddr, mfn, access, bdf, size);
+
+    /* the interface allows it but the current implementation doesn't */
+    BUG_ON(retaddr != ioaddr);
+
+    return retaddr;
+}
+
+/* unmap takes a single "entry" - use hypercalls for batching */
+int do_iommu_unmap(u64 ioaddr, u32 bdf, u32 size)
+{
+    void* space;
+    int ret;
+
+    space = find_io_space(bdf);
+
+    ret = calgary_unmap(space, ioaddr, bdf, size);
+
+    return ret;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/common/Makefile 
xen/xen/common/Makefile
--- vanilla.xen/xen/common/Makefile     2006-03-19 20:10:13.000000000 +0200
+++ xen/xen/common/Makefile     2006-03-20 12:28:54.000000000 +0200
@@ -2,11 +2,13 @@ include $(BASEDIR)/Rules.mk
 
 obj-y += acm_ops.o
 obj-y += bitmap.o
+obj-y += calgary.o
 obj-y += dom0_ops.o
 obj-y += domain.o
 obj-y += elf.o
 obj-y += event_channel.o
 obj-y += grant_table.o
+obj-y += iommu.o
 obj-y += kernel.o
 obj-y += keyhandler.o
 obj-y += lib.o
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/include/public/dom0_ops.h 
xen/xen/include/public/dom0_ops.h
--- vanilla.xen/xen/include/public/dom0_ops.h   2006-03-07 19:54:46.000000000 
+0200
+++ xen/xen/include/public/dom0_ops.h   2006-03-07 20:35:04.000000000 +0200
@@ -470,6 +470,20 @@ typedef struct dom0_hypercall_init {
     unsigned long mfn;        /* machine frame to be initialised */
 } dom0_hypercall_init_t;
 DEFINE_GUEST_HANDLE(dom0_hypercall_init_t);
+ 
+#define DOM0_IOMMU_CREATE_IO_SPACE 49
+typedef struct dom0_iommu_create_io_space {
+    u64 size;       /* size of the IO address space in bytes */
+    u32 bdf;        /* bus/dev/func this space translates for */
+    u64 rootmfn;    /* mfn of the root of the iommu table */
+} dom0_iommu_create_io_space_t;
+DEFINE_GUEST_HANDLE(dom0_iommu_create_io_space_t);
+
+#define DOM0_IOMMU_DESTROY_IO_SPACE 50
+typedef struct dom0_iommu_destroy_io_space {
+    u32 bdf;
+} dom0_iommu_destroy_io_space_t;
+DEFINE_GUEST_HANDLE(dom0_iommu_destroy_io_space_t);
 
 typedef struct dom0_op {
     uint32_t cmd;
@@ -512,6 +526,8 @@ typedef struct dom0_op {
         struct dom0_irq_permission    irq_permission;
         struct dom0_iomem_permission  iomem_permission;
         struct dom0_hypercall_init    hypercall_init;
+        struct dom0_iommu_create_io_space  iommu_create_io_space;
+        struct dom0_iommu_destroy_io_space iommu_destroy_io_space;
         uint8_t                       pad[128];
     } u;
 } dom0_op_t;
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/include/public/xen.h 
xen/xen/include/public/xen.h
--- vanilla.xen/xen/include/public/xen.h        2006-03-16 09:53:07.000000000 
+0200
+++ xen/xen/include/public/xen.h        2006-03-16 23:08:13.000000000 +0200
@@ -60,6 +60,8 @@
 #define __HYPERVISOR_acm_op               27
 #define __HYPERVISOR_nmi_op               28
 #define __HYPERVISOR_sched_op_new         29
+#define __HYPERVISOR_iommu_map            30
+#define __HYPERVISOR_iommu_unmap          31
 
 /* 
  * VIRTUAL INTERRUPTS
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/include/xen/calgary.h 
xen/xen/include/xen/calgary.h
--- vanilla.xen/xen/include/xen/calgary.h       1970-01-01 02:00:00.000000000 
+0200
+++ xen/xen/include/xen/calgary.h       2006-03-16 23:22:27.000000000 +0200
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2006 Muli Ben-Yehuda <mulix@xxxxxxxxx>, IBM Corporation
+ */
+
+#define TCE_MAX_TABLE_SIZE 7
+
+struct iommu_table {
+    u32 size; /* in number of entries */
+    void* ptr;
+};
+
+#define TCE_ENTRY_SIZE   8   /* in bytes */
+
+#define TCE_READ_SHIFT   0
+#define TCE_WRITE_SHIFT  1
+#define TCE_HUBID_SHIFT  2   /* unused */
+#define TCE_RSVD_SHIFT   8   /* unused */
+#define TCE_RPN_SHIFT    12
+#define TCE_UNUSED_SHIFT 48  /* unused */
+
+#define TCE_RPN_MASK     0x0000fffffffff000ULL
+
+int calgary_create_io_space(u64 size, u32 bdf, void** space, u64* rootmfn);
+void calgary_destroy_io_space(void* space);
+u64 calgary_map(void* space, u64 ioaddr, u64 mfn, u32 access, u32 bdf,
+        u32 size);
+int calgary_unmap(void* space, u64 ioaddr, u32 bdf, u32 size);
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -Naurp -X /home/muli/w/dontdiff vanilla.xen/xen/include/xen/iommu.h 
xen/xen/include/xen/iommu.h
--- vanilla.xen/xen/include/xen/iommu.h 1970-01-01 02:00:00.000000000 +0200
+++ xen/xen/include/xen/iommu.h 2006-03-20 15:17:11.000000000 +0200
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2006 Muli Ben-Yehuda <mulix@xxxxxxxxx>, IBM Corporation
+ */
+
+#include <asm/types.h>
+#include <asm/page.h>
+#include <public/dom0_ops.h>
+
+int iommu_create_io_space(GUEST_HANDLE(dom0_op_t) u_dom0_op,
+    struct dom0_op *op);
+int iommu_destroy_io_space(u32 bdf);
+u64 do_iommu_map(u64 ioaddr, u64 mfn, u32 access, u32 bdf, u32 size);
+int do_iommu_unmap(u64 ioaddr, u32 bdf, u32 size);
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */

-- 
Muli Ben-Yehuda
http://www.mulix.org | http://mulix.livejournal.com/


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel

<Prev in Thread] Current Thread [Next in Thread>