diff -r 5e1df44d406e xen/arch/x86/hvm/svm/amd_iommu/amd-iommu-detect.c --- a/xen/arch/x86/hvm/svm/amd_iommu/amd-iommu-detect.c Wed Feb 13 14:03:58 2008 +0000 +++ b/xen/arch/x86/hvm/svm/amd_iommu/amd-iommu-detect.c Wed Feb 13 16:44:26 2008 +0100 @@ -89,12 +89,14 @@ int __init get_iommu_capabilities(u8 bus u32 cap_header, cap_range; u64 mmio_bar; +#if HACK_BIOS_SETTINGS /* remove it when BIOS available */ write_pci_config(bus, dev, func, cap_ptr + PCI_CAP_MMIO_BAR_HIGH_OFFSET, 0x00000000); write_pci_config(bus, dev, func, cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET, 0x40000001); /* remove it when BIOS available */ +#endif mmio_bar = (u64)read_pci_config(bus, dev, func, cap_ptr + PCI_CAP_MMIO_BAR_HIGH_OFFSET) << 32; diff -r 5e1df44d406e xen/arch/x86/hvm/svm/amd_iommu/amd-iommu-map.c --- a/xen/arch/x86/hvm/svm/amd_iommu/amd-iommu-map.c Wed Feb 13 14:03:58 2008 +0000 +++ b/xen/arch/x86/hvm/svm/amd_iommu/amd-iommu-map.c Wed Feb 13 16:44:26 2008 +0100 @@ -113,7 +113,7 @@ static void invalidate_iommu_page(struct send_iommu_command(iommu, cmd); } -static void flush_command_buffer(struct amd_iommu *iommu) +void flush_command_buffer(struct amd_iommu *iommu) { u32 cmd[4], status; int loop_count, comp_wait; @@ -278,7 +278,7 @@ void amd_iommu_set_dev_table_entry(u32 * dte[0] = entry; } -static void *amd_iommu_get_vptr_from_page_table_entry(u32 *entry) +void *amd_iommu_get_vptr_from_page_table_entry(u32 *entry) { u64 addr_lo, addr_hi, ptr; @@ -299,6 +299,34 @@ static int amd_iommu_is_pte_present(u32 return (get_field_from_reg_u32(entry[0], IOMMU_PDE_PRESENT_MASK, IOMMU_PDE_PRESENT_SHIFT)); +} + +void invalidate_dev_table_entry(struct amd_iommu *iommu, + u16 device_id) +{ + u32 cmd[4], entry; + + cmd[3] = cmd[2] = 0; + set_field_in_reg_u32(device_id, 0, + IOMMU_INV_DEVTAB_ENTRY_DEVICE_ID_MASK, + IOMMU_INV_DEVTAB_ENTRY_DEVICE_ID_SHIFT, &entry); + cmd[0] = entry; + + set_field_in_reg_u32(IOMMU_CMD_INVALIDATE_DEVTAB_ENTRY, 0, + IOMMU_CMD_OPCODE_MASK, IOMMU_CMD_OPCODE_SHIFT, &entry); + cmd[1] = entry; + + send_iommu_command(iommu, cmd); +} + +int amd_iommu_is_dte_page_translation_valid(u32 *entry) +{ + return (get_field_from_reg_u32(entry[0], + IOMMU_DEV_TABLE_VALID_MASK, + IOMMU_DEV_TABLE_VALID_SHIFT) && + get_field_from_reg_u32(entry[0], + IOMMU_DEV_TABLE_TRANSLATION_VALID_MASK, + IOMMU_DEV_TABLE_TRANSLATION_VALID_SHIFT)); } static void *get_pte_from_page_tables(void *table, int level, diff -r 5e1df44d406e xen/arch/x86/hvm/svm/amd_iommu/pci-amd-iommu.c --- a/xen/arch/x86/hvm/svm/amd_iommu/pci-amd-iommu.c Wed Feb 13 14:03:58 2008 +0000 +++ b/xen/arch/x86/hvm/svm/amd_iommu/pci-amd-iommu.c Wed Feb 13 16:44:26 2008 +0100 @@ -76,11 +76,11 @@ static void __init deallocate_iommu_reso static void __init detect_cleanup(void) { - struct amd_iommu *iommu; + struct amd_iommu *iommu, *next; dprintk(XENLOG_ERR, "AMD IOMMU: %s()\n", __FUNCTION__); - for_each_amd_iommu(iommu) { + list_for_each_entry_safe(iommu, next, &amd_iommu_head, list) { list_del(&iommu->list); deallocate_iommu_resources(iommu); xfree(iommu); @@ -238,16 +238,20 @@ void amd_iommu_setup_domain_device( dte = iommu->dev_table.buffer + (requestor_id * IOMMU_DEV_TABLE_ENTRY_SIZE); - spin_lock_irqsave(&iommu->lock, flags); - - amd_iommu_set_dev_table_entry((u32 *)dte, - root_ptr, hd->domain_id, hd->paging_mode); - - dprintk(XENLOG_INFO, "AMD IOMMU: Set DTE req_id:%x, " + if ( !(amd_iommu_is_dte_page_translation_valid((u32 *)dte)) ) + { + spin_lock_irqsave(&iommu->lock, flags); + + amd_iommu_set_dev_table_entry((u32 *)dte, + root_ptr, hd->domain_id, hd->paging_mode); + invalidate_dev_table_entry(iommu, requestor_id); + flush_command_buffer(iommu); + dprintk(XENLOG_INFO, "AMD IOMMU: Set DTE req_id:%x, " "root_ptr:%"PRIx64", domain_id:%d, paging_mode:%d\n", requestor_id, root_ptr, hd->domain_id, hd->paging_mode); - spin_unlock_irqrestore(&iommu->lock, flags); + spin_unlock_irqrestore(&iommu->lock, flags); + } } void __init amd_iommu_setup_dom0_devices(void) @@ -310,7 +314,7 @@ int amd_iommu_detect(void) goto error_out; } - if ( amd_iommu_domain_init(dom0) != 0 ) + if ( iommu_domain_init(dom0) != 0 ) goto error_out; /* setup 1:1 page table for dom0 */ @@ -329,12 +333,22 @@ static int allocate_domain_resources(str static int allocate_domain_resources(struct hvm_iommu *hd) { /* allocate root table */ - hd->root_table = (void *)alloc_xenheap_page(); + unsigned long flags; + + spin_lock_irqsave(&hd->mapping_lock, flags); if ( !hd->root_table ) - return -ENOMEM; - memset((u8*)hd->root_table, 0, PAGE_SIZE); - - return 0; + { + hd->root_table = (void *)alloc_xenheap_page(); + if ( !hd->root_table ) + goto error_out; + memset((u8*)hd->root_table, 0, PAGE_SIZE); + } + spin_unlock_irqrestore(&hd->mapping_lock, flags); + + return 0; +error_out: + spin_unlock_irqrestore(&hd->mapping_lock, flags); + return -ENOMEM; } static int get_paging_mode(unsigned long entries) @@ -362,10 +376,6 @@ int amd_iommu_domain_init(struct domain { struct hvm_iommu *hd = domain_hvm_iommu(domain); - spin_lock_init(&hd->mapping_lock); - spin_lock_init(&hd->iommu_list_lock); - INIT_LIST_HEAD(&hd->pdev_list); - /* allocate page directroy */ if ( allocate_domain_resources(hd) != 0 ) { dprintk(XENLOG_ERR, "AMD IOMMU: %s()\n", __FUNCTION__); @@ -386,4 +396,166 @@ error_out: return -ENOMEM; } - +static void amd_iommu_disable_domain_device( + struct domain *domain, struct amd_iommu *iommu, u16 requestor_id) +{ + void *dte; + unsigned long flags; + + dte = iommu->dev_table.buffer + + (requestor_id * IOMMU_DEV_TABLE_ENTRY_SIZE); + + if ( amd_iommu_is_dte_page_translation_valid((u32 *)dte) ) + { + spin_lock_irqsave(&iommu->lock, flags); + memset (dte, 0, IOMMU_DEV_TABLE_ENTRY_SIZE); + invalidate_dev_table_entry(iommu, requestor_id); + flush_command_buffer(iommu); + dprintk(XENLOG_INFO , "AMD IOMMU: disable DTE 0x%x," + " domain_id:%d, paging_mode:%d\n", + requestor_id, domain_hvm_iommu(domain)->domain_id, + domain_hvm_iommu(domain)->paging_mode); + spin_unlock_irqrestore(&iommu->lock, flags); + } +} + +extern void pdev_flr(u8 bus, u8 devfn); + +static int reassign_device( struct domain *source, struct domain *target, + u8 bus, u8 devfn) +{ + struct hvm_iommu *source_hd = domain_hvm_iommu(source); + struct hvm_iommu *target_hd = domain_hvm_iommu(target); + struct pci_dev *pdev; + struct amd_iommu *iommu; + int req_id, bdf; + unsigned long flags; + + for_each_pdev( source, pdev ) + { + if ( (pdev->bus != bus) || (pdev->devfn != devfn) ) + continue; + + pdev->bus = bus; + pdev->devfn = devfn; + + bdf = (bus << 8) | devfn; + req_id = requestor_id_from_bdf(bdf); + iommu = find_iommu_for_device(bus, devfn); + + if ( iommu ) + { + amd_iommu_disable_domain_device(source, iommu, req_id); + /* Move pci device from the source domain to target domain. */ + spin_lock_irqsave(&source_hd->iommu_list_lock, flags); + spin_lock_irqsave(&target_hd->iommu_list_lock, flags); + list_move(&pdev->list, &target_hd->pdev_list); + spin_unlock_irqrestore(&target_hd->iommu_list_lock, flags); + spin_unlock_irqrestore(&source_hd->iommu_list_lock, flags); + + amd_iommu_setup_domain_device(target, iommu, req_id); + gdprintk(XENLOG_INFO , + "AMD IOMMU: reassign %x:%x.%x domain %d -> domain %d\n", + bus, PCI_SLOT(devfn), PCI_FUNC(devfn), + source->domain_id, target->domain_id); + } + else + { + gdprintk(XENLOG_ERR , "AMD IOMMU: fail to find iommu." + " %x:%x.%x cannot be assigned to domain %d\n", + bus, PCI_SLOT(devfn), PCI_FUNC(devfn), target->domain_id); + return -ENODEV; + } + + break; + } + return 0; +} + +int amd_iommu_assign_device(struct domain *d, u8 bus, u8 devfn) +{ + pdev_flr(bus, devfn); + return reassign_device(dom0, d, bus, devfn); +} + +static void release_domain_devices(struct domain *d) +{ + struct hvm_iommu *hd = domain_hvm_iommu(d); + struct pci_dev *pdev; + + while ( !list_empty(&hd->pdev_list) ) + { + pdev = list_entry(hd->pdev_list.next, typeof(*pdev), list); + pdev_flr(pdev->bus, pdev->devfn); + gdprintk(XENLOG_INFO , + "AMD IOMMU: release devices %x:%x.%x\n", + pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)); + reassign_device(d, dom0, pdev->bus, pdev->devfn); + } +} + +static void deallocate_next_page_table(void *table, unsigned long index, + int level) +{ + unsigned long next_index; + void *next_table, *pde; + int next_level; + + pde = table + (index * IOMMU_PAGE_TABLE_ENTRY_SIZE); + next_table = amd_iommu_get_vptr_from_page_table_entry((u32 *)pde); + + if ( next_table ) + { + next_level = level - 1; + if ( next_level > 1 ) + { + next_index = 0; + do + { + deallocate_next_page_table(next_table, + next_index, next_level); + ++next_index; + } while (next_index < PTE_PER_TABLE_SIZE); + } + + free_xenheap_page(next_table); + } +} + +static void deallocate_iommu_page_tables(struct domain *d) +{ + unsigned long index; + struct hvm_iommu *hd = domain_hvm_iommu(d); + + if ( hd ->root_table ) + { + index = 0; + do + { + deallocate_next_page_table(hd->root_table, + index, hd->paging_mode); + ++index; + } while ( index < PTE_PER_TABLE_SIZE ); + + free_xenheap_page(hd ->root_table); + } + + hd ->root_table = NULL; +} + +void amd_iommu_domain_destroy(struct domain *d) +{ + if ( !amd_iommu_enabled ) + return; + + deallocate_iommu_page_tables(d); + release_domain_devices(d); +} + +struct iommu_ops amd_iommu_ops = { + .init = amd_iommu_domain_init, + .assign_device = amd_iommu_assign_device, + .teardown = amd_iommu_domain_destroy, + .map_page = amd_iommu_map_page, + .unmap_page = amd_iommu_unmap_page, +}; diff -r 5e1df44d406e xen/arch/x86/hvm/svm/intr.c --- a/xen/arch/x86/hvm/svm/intr.c Wed Feb 13 14:03:58 2008 +0000 +++ b/xen/arch/x86/hvm/svm/intr.c Wed Feb 13 16:44:26 2008 +0100 @@ -94,6 +94,46 @@ static void enable_intr_window(struct vc vmcb->general1_intercepts |= GENERAL1_INTERCEPT_VINTR; } +static void svm_dirq_assist(struct vcpu *v) +{ + unsigned int irq; + uint32_t device, intx; + struct domain *d = v->domain; + struct hvm_irq_dpci *hvm_irq_dpci = d->arch.hvm_domain.irq.dpci; + struct dev_intx_gsi_link *digl; + + if ( !amd_iommu_enabled || (v->vcpu_id != 0) || (hvm_irq_dpci == NULL) ) + return; + + for ( irq = find_first_bit(hvm_irq_dpci->dirq_mask, NR_IRQS); + irq < NR_IRQS; + irq = find_next_bit(hvm_irq_dpci->dirq_mask, NR_IRQS, irq + 1) ) + { + stop_timer(&hvm_irq_dpci->hvm_timer[irq_to_vector(irq)]); + clear_bit(irq, &hvm_irq_dpci->dirq_mask); + + list_for_each_entry ( digl, &hvm_irq_dpci->mirq[irq].digl_list, list ) + { + device = digl->device; + intx = digl->intx; + hvm_pci_intx_assert(d, device, intx); + spin_lock(&hvm_irq_dpci->dirq_lock); + hvm_irq_dpci->mirq[irq].pending++; + spin_unlock(&hvm_irq_dpci->dirq_lock); + } + + /* + * Set a timer to see if the guest can finish the interrupt or not. For + * example, the guest OS may unmask the PIC during boot, before the + * guest driver is loaded. hvm_pci_intx_assert() may succeed, but the + * guest will never deal with the irq, then the physical interrupt line + * will never be deasserted. + */ + set_timer(&hvm_irq_dpci->hvm_timer[irq_to_vector(irq)], + NOW() + PT_IRQ_TIME_OUT); + } +} + asmlinkage void svm_intr_assist(void) { struct vcpu *v = current; @@ -102,6 +142,7 @@ asmlinkage void svm_intr_assist(void) /* Crank the handle on interrupt state. */ pt_update_irq(v); + svm_dirq_assist(v); do { intack = hvm_vcpu_has_pending_irq(v); diff -r 5e1df44d406e xen/include/asm-x86/hvm/svm/amd-iommu-defs.h --- a/xen/include/asm-x86/hvm/svm/amd-iommu-defs.h Wed Feb 13 14:03:58 2008 +0000 +++ b/xen/include/asm-x86/hvm/svm/amd-iommu-defs.h Wed Feb 13 16:44:26 2008 +0100 @@ -262,6 +262,10 @@ #define IOMMU_INV_IOMMU_PAGES_ADDR_LOW_SHIFT 12 #define IOMMU_INV_IOMMU_PAGES_ADDR_HIGH_MASK 0xFFFFFFFF #define IOMMU_INV_IOMMU_PAGES_ADDR_HIGH_SHIFT 0 + +/* INVALIDATE_DEVTAB_ENTRY command */ +#define IOMMU_INV_DEVTAB_ENTRY_DEVICE_ID_MASK 0x0000FFFF +#define IOMMU_INV_DEVTAB_ENTRY_DEVICE_ID_SHIFT 0 /* Event Log */ #define IOMMU_EVENT_LOG_BASE_LOW_OFFSET 0x10 @@ -415,5 +419,6 @@ #define IOMMU_PAGE_TABLE_LEVEL_4 4 #define IOMMU_IO_WRITE_ENABLED 1 #define IOMMU_IO_READ_ENABLED 1 +#define HACK_BIOS_SETTINGS 0 #endif /* _ASM_X86_64_AMD_IOMMU_DEFS_H */ diff -r 5e1df44d406e xen/include/asm-x86/hvm/svm/amd-iommu-proto.h --- a/xen/include/asm-x86/hvm/svm/amd-iommu-proto.h Wed Feb 13 14:03:58 2008 +0000 +++ b/xen/include/asm-x86/hvm/svm/amd-iommu-proto.h Wed Feb 13 16:44:26 2008 +0100 @@ -27,6 +27,10 @@ list_for_each_entry(amd_iommu, \ &amd_iommu_head, list) +#define for_each_pdev(domain, pdev) \ + list_for_each_entry(pdev, \ + &(domain->arch.hvm_domain.hvm_iommu.pdev_list), list) + #define DMA_32BIT_MASK 0x00000000ffffffffULL #define PAGE_ALIGN(addr) (((addr) + PAGE_SIZE - 1) & PAGE_MASK) #define PAGE_SHIFT_4K (12) @@ -52,13 +56,18 @@ int amd_iommu_map_page(struct domain *d, int amd_iommu_map_page(struct domain *d, unsigned long gfn, unsigned long mfn); int amd_iommu_unmap_page(struct domain *d, unsigned long gfn); +void *amd_iommu_get_vptr_from_page_table_entry(u32 *entry); /* device table functions */ void amd_iommu_set_dev_table_entry(u32 *dte, u64 root_ptr, u16 domain_id, u8 paging_mode); +int amd_iommu_is_dte_page_translation_valid(u32 *entry); +void invalidate_dev_table_entry(struct amd_iommu *iommu, + u16 devic_id); /* send cmd to iommu */ int send_iommu_command(struct amd_iommu *iommu, u32 cmd[]); +void flush_command_buffer(struct amd_iommu *iommu); /* iommu domain funtions */ int amd_iommu_domain_init(struct domain *domain);