|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] Re: [PATCH v2 2/8] xen/arm: revert atomic operation related command-queue insertion patch
Hello Stefano,
Thanks for reviewing the code.
> On 1 Dec 2020, at 10:23 pm, Stefano Stabellini <sstabellini@xxxxxxxxxx> wrote:
>
> On Thu, 26 Nov 2020, Rahul Singh wrote:
>> Linux SMMUv3 code implements the commands-queue insertion based on
>> atomic operations implemented in Linux. Atomic functions used by the
>> commands-queue insertion is not implemented in XEN therefore revert the
>> patch that implemented the commands-queue insertion based on atomic
>> operations.
>>
>> Once the proper atomic operations will be available in XEN the driver
>> can be updated.
>>
>> Reverted the commit 587e6c10a7ce89a5924fdbeff2ec524fbd6a124b
>> iommu/arm-smmu-v3: Reduce contention during command-queue insertion
>
> I checked 587e6c10a7ce89a5924fdbeff2ec524fbd6a124b: this patch does more
> than just reverting 587e6c10a7ce89a5924fdbeff2ec524fbd6a124b. It looks
> like it is also reverting edd0351e7bc49555d8b5ad8438a65a7ca262c9f0 and
> some other commits.
>
> Please can you provide a complete list of reverted commits? I would like
> to be able to do the reverts myself on the linux tree and see that the
> driver textually matches the one on the xen tree with this patch
> applied.
>
>
Yes this patch is also reverting the commits that is based on the code that
introduced the atomic-operations. I will add all the commit id in next version
of the patch in commit msg.
Patches that are reverted in this patch are as follows:
9e773aee8c3e1b3ba019c5c7f8435aaa836c6130 iommu/arm-smmu-v3: Batch ATC
invalidation commands
edd0351e7bc49555d8b5ad8438a65a7ca262c9f0 iommu/arm-smmu-v3: Batch context
descriptor invalidation
4ce8da453640147101bda418640394637c1a7cfc iommu/arm-smmu-v3: Add command queue
batching helpers
2af2e72b18b499fa36d3f7379fd010ff25d2a984. iommu/arm-smmu-v3: Defer TLB
invalidation until ->iotlb_sync()
587e6c10a7ce89a5924fdbeff2ec524fbd6a124b iommu/arm-smmu-v3: Reduce
contention during command-queue insertion
Regards,
Rahul
>
>> Signed-off-by: Rahul Singh <rahul.singh@xxxxxxx>
>> ---
>> xen/drivers/passthrough/arm/smmu-v3.c | 847 ++++++--------------------
>> 1 file changed, 180 insertions(+), 667 deletions(-)
>>
>> diff --git a/xen/drivers/passthrough/arm/smmu-v3.c
>> b/xen/drivers/passthrough/arm/smmu-v3.c
>> index c192544e87..97eac61ea4 100644
>> --- a/xen/drivers/passthrough/arm/smmu-v3.c
>> +++ b/xen/drivers/passthrough/arm/smmu-v3.c
>> @@ -330,15 +330,6 @@
>> #define CMDQ_ERR_CERROR_ABT_IDX 2
>> #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
>>
>> -#define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
>> -
>> -/*
>> - * This is used to size the command queue and therefore must be at least
>> - * BITS_PER_LONG so that the valid_map works correctly (it relies on the
>> - * total number of queue entries being a multiple of BITS_PER_LONG).
>> - */
>> -#define CMDQ_BATCH_ENTRIES BITS_PER_LONG
>> -
>> #define CMDQ_0_OP GENMASK_ULL(7, 0)
>> #define CMDQ_0_SSV (1UL << 11)
>>
>> @@ -407,8 +398,9 @@
>> #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
>>
>> /* High-level queue structures */
>> -#define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
>> -#define ARM_SMMU_POLL_SPIN_COUNT 10
>> +#define ARM_SMMU_POLL_TIMEOUT_US 100
>> +#define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US 1000000 /* 1s! */
>> +#define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT 10
>>
>> #define MSI_IOVA_BASE 0x8000000
>> #define MSI_IOVA_LENGTH 0x100000
>> @@ -513,24 +505,15 @@ struct arm_smmu_cmdq_ent {
>>
>> #define CMDQ_OP_CMD_SYNC 0x46
>> struct {
>> + u32 msidata;
>> u64 msiaddr;
>> } sync;
>> };
>> };
>>
>> struct arm_smmu_ll_queue {
>> - union {
>> - u64 val;
>> - struct {
>> - u32 prod;
>> - u32 cons;
>> - };
>> - struct {
>> - atomic_t prod;
>> - atomic_t cons;
>> - } atomic;
>> - u8 __pad[SMP_CACHE_BYTES];
>> - } ____cacheline_aligned_in_smp;
>> + u32 prod;
>> + u32 cons;
>> u32 max_n_shift;
>> };
>>
>> @@ -548,23 +531,9 @@ struct arm_smmu_queue {
>> u32 __iomem *cons_reg;
>> };
>>
>> -struct arm_smmu_queue_poll {
>> - ktime_t timeout;
>> - unsigned int delay;
>> - unsigned int spin_cnt;
>> - bool wfe;
>> -};
>> -
>> struct arm_smmu_cmdq {
>> struct arm_smmu_queue q;
>> - atomic_long_t *valid_map;
>> - atomic_t owner_prod;
>> - atomic_t lock;
>> -};
>> -
>> -struct arm_smmu_cmdq_batch {
>> - u64 cmds[CMDQ_BATCH_ENTRIES *
>> CMDQ_ENT_DWORDS];
>> - int num;
>> + spinlock_t lock;
>> };
>>
>> struct arm_smmu_evtq {
>> @@ -660,6 +629,8 @@ struct arm_smmu_device {
>>
>> int gerr_irq;
>> int combined_irq;
>> + u32 sync_nr;
>> + u8 prev_cmd_opcode;
>>
>> unsigned long ias; /* IPA */
>> unsigned long oas; /* PA */
>> @@ -677,6 +648,12 @@ struct arm_smmu_device {
>>
>> struct arm_smmu_strtab_cfg strtab_cfg;
>>
>> + /* Hi16xx adds an extra 32 bits of goodness to its MSI payload */
>> + union {
>> + u32 sync_count;
>> + u64 padding;
>> + };
>> +
>> /* IOMMU core code handle */
>> struct iommu_device iommu;
>> };
>> @@ -763,21 +740,6 @@ static void parse_driver_options(struct arm_smmu_device
>> *smmu)
>> }
>>
>> /* Low-level queue manipulation functions */
>> -static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
>> -{
>> - u32 space, prod, cons;
>> -
>> - prod = Q_IDX(q, q->prod);
>> - cons = Q_IDX(q, q->cons);
>> -
>> - if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
>> - space = (1 << q->max_n_shift) - (prod - cons);
>> - else
>> - space = cons - prod;
>> -
>> - return space >= n;
>> -}
>> -
>> static bool queue_full(struct arm_smmu_ll_queue *q)
>> {
>> return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
>> @@ -790,12 +752,9 @@ static bool queue_empty(struct arm_smmu_ll_queue *q)
>> Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
>> }
>>
>> -static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
>> +static void queue_sync_cons_in(struct arm_smmu_queue *q)
>> {
>> - return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
>> - (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
>> - ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
>> - (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
>> + q->llq.cons = readl_relaxed(q->cons_reg);
>> }
>>
>> static void queue_sync_cons_out(struct arm_smmu_queue *q)
>> @@ -826,34 +785,46 @@ static int queue_sync_prod_in(struct arm_smmu_queue *q)
>> return ret;
>> }
>>
>> -static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
>> +static void queue_sync_prod_out(struct arm_smmu_queue *q)
>> {
>> - u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
>> - return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
>> + writel(q->llq.prod, q->prod_reg);
>> }
>>
>> -static void queue_poll_init(struct arm_smmu_device *smmu,
>> - struct arm_smmu_queue_poll *qp)
>> +static void queue_inc_prod(struct arm_smmu_ll_queue *q)
>> {
>> - qp->delay = 1;
>> - qp->spin_cnt = 0;
>> - qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
>> - qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
>> + u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
>> + q->prod = Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
>> }
>>
>> -static int queue_poll(struct arm_smmu_queue_poll *qp)
>> +/*
>> + * Wait for the SMMU to consume items. If sync is true, wait until the queue
>> + * is empty. Otherwise, wait until there is at least one free slot.
>> + */
>> +static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
>> {
>> - if (ktime_compare(ktime_get(), qp->timeout) > 0)
>> - return -ETIMEDOUT;
>> + ktime_t timeout;
>> + unsigned int delay = 1, spin_cnt = 0;
>>
>> - if (qp->wfe) {
>> - wfe();
>> - } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
>> - cpu_relax();
>> - } else {
>> - udelay(qp->delay);
>> - qp->delay *= 2;
>> - qp->spin_cnt = 0;
>> + /* Wait longer if it's a CMD_SYNC */
>> + timeout = ktime_add_us(ktime_get(), sync ?
>> + ARM_SMMU_CMDQ_SYNC_TIMEOUT_US :
>> + ARM_SMMU_POLL_TIMEOUT_US);
>> +
>> + while (queue_sync_cons_in(q),
>> + (sync ? !queue_empty(&q->llq) : queue_full(&q->llq))) {
>> + if (ktime_compare(ktime_get(), timeout) > 0)
>> + return -ETIMEDOUT;
>> +
>> + if (wfe) {
>> + wfe();
>> + } else if (++spin_cnt < ARM_SMMU_CMDQ_SYNC_SPIN_COUNT) {
>> + cpu_relax();
>> + continue;
>> + } else {
>> + udelay(delay);
>> + delay *= 2;
>> + spin_cnt = 0;
>> + }
>> }
>>
>> return 0;
>> @@ -867,6 +838,17 @@ static void queue_write(__le64 *dst, u64 *src, size_t
>> n_dwords)
>> *dst++ = cpu_to_le64(*src++);
>> }
>>
>> +static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
>> +{
>> + if (queue_full(&q->llq))
>> + return -ENOSPC;
>> +
>> + queue_write(Q_ENT(q, q->llq.prod), ent, q->ent_dwords);
>> + queue_inc_prod(&q->llq);
>> + queue_sync_prod_out(q);
>> + return 0;
>> +}
>> +
>> static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
>> {
>> int i;
>> @@ -964,14 +946,20 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
>> arm_smmu_cmdq_ent *ent)
>> cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
>> break;
>> case CMDQ_OP_CMD_SYNC:
>> - if (ent->sync.msiaddr) {
>> + if (ent->sync.msiaddr)
>> cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS,
>> CMDQ_SYNC_0_CS_IRQ);
>> - cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
>> - } else {
>> + else
>> cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS,
>> CMDQ_SYNC_0_CS_SEV);
>> - }
>> cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
>> cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR,
>> ARM_SMMU_MEMATTR_OIWB);
>> + /*
>> + * Commands are written little-endian, but we want the SMMU to
>> + * receive MSIData, and thus write it back to memory, in CPU
>> + * byte order, so big-endian needs an extra byteswap here.
>> + */
>> + cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIDATA,
>> + cpu_to_le32(ent->sync.msidata));
>> + cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
>> break;
>> default:
>> return -ENOENT;
>> @@ -980,27 +968,6 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
>> arm_smmu_cmdq_ent *ent)
>> return 0;
>> }
>>
>> -static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device
>> *smmu,
>> - u32 prod)
>> -{
>> - struct arm_smmu_queue *q = &smmu->cmdq.q;
>> - struct arm_smmu_cmdq_ent ent = {
>> - .opcode = CMDQ_OP_CMD_SYNC,
>> - };
>> -
>> - /*
>> - * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
>> - * payload, so the write will zero the entire command on that platform.
>> - */
>> - if (smmu->features & ARM_SMMU_FEAT_MSI &&
>> - smmu->features & ARM_SMMU_FEAT_COHERENCY) {
>> - ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
>> - q->ent_dwords * 8;
>> - }
>> -
>> - arm_smmu_cmdq_build_cmd(cmd, &ent);
>> -}
>> -
>> static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
>> {
>> static const char *cerror_str[] = {
>> @@ -1058,474 +1025,109 @@ static void arm_smmu_cmdq_skip_err(struct
>> arm_smmu_device *smmu)
>> queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
>> }
>>
>> -/*
>> - * Command queue locking.
>> - * This is a form of bastardised rwlock with the following major changes:
>> - *
>> - * - The only LOCK routines are exclusive_trylock() and shared_lock().
>> - * Neither have barrier semantics, and instead provide only a control
>> - * dependency.
>> - *
>> - * - The UNLOCK routines are supplemented with shared_tryunlock(), which
>> - * fails if the caller appears to be the last lock holder (yes, this is
>> - * racy). All successful UNLOCK routines have RELEASE semantics.
>> - */
>> -static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
>> +static void arm_smmu_cmdq_insert_cmd(struct arm_smmu_device *smmu, u64 *cmd)
>> {
>> - int val;
>> -
>> - /*
>> - * We can try to avoid the cmpxchg() loop by simply incrementing the
>> - * lock counter. When held in exclusive state, the lock counter is set
>> - * to INT_MIN so these increments won't hurt as the value will remain
>> - * negative.
>> - */
>> - if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
>> - return;
>> -
>> - do {
>> - val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
>> - } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
>> -}
>> -
>> -static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
>> -{
>> - (void)atomic_dec_return_release(&cmdq->lock);
>> -}
>> -
>> -static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
>> -{
>> - if (atomic_read(&cmdq->lock) == 1)
>> - return false;
>> -
>> - arm_smmu_cmdq_shared_unlock(cmdq);
>> - return true;
>> -}
>> -
>> -#define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)
>> \
>> -({ \
>> - bool __ret; \
>> - local_irq_save(flags); \
>> - __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
>> - if (!__ret) \
>> - local_irq_restore(flags); \
>> - __ret; \
>> -})
>> -
>> -#define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)
>> \
>> -({ \
>> - atomic_set_release(&cmdq->lock, 0); \
>> - local_irq_restore(flags); \
>> -})
>> -
>> -
>> -/*
>> - * Command queue insertion.
>> - * This is made fiddly by our attempts to achieve some sort of scalability
>> - * since there is one queue shared amongst all of the CPUs in the system.
>> If
>> - * you like mixed-size concurrency, dependency ordering and relaxed atomics,
>> - * then you'll *love* this monstrosity.
>> - *
>> - * The basic idea is to split the queue up into ranges of commands that are
>> - * owned by a given CPU; the owner may not have written all of the commands
>> - * itself, but is responsible for advancing the hardware prod pointer when
>> - * the time comes. The algorithm is roughly:
>> - *
>> - * 1. Allocate some space in the queue. At this point we also discover
>> - * whether the head of the queue is currently owned by another CPU,
>> - * or whether we are the owner.
>> - *
>> - * 2. Write our commands into our allocated slots in the queue.
>> - *
>> - * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
>> - *
>> - * 4. If we are an owner:
>> - * a. Wait for the previous owner to finish.
>> - * b. Mark the queue head as unowned, which tells us the range
>> - * that we are responsible for publishing.
>> - * c. Wait for all commands in our owned range to become valid.
>> - * d. Advance the hardware prod pointer.
>> - * e. Tell the next owner we've finished.
>> - *
>> - * 5. If we are inserting a CMD_SYNC (we may or may not have been an
>> - * owner), then we need to stick around until it has completed:
>> - * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
>> - * to clear the first 4 bytes.
>> - * b. Otherwise, we spin waiting for the hardware cons pointer to
>> - * advance past our command.
>> - *
>> - * The devil is in the details, particularly the use of locking for handling
>> - * SYNC completion and freeing up space in the queue before we think that
>> it is
>> - * full.
>> - */
>> -static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
>> - u32 sprod, u32 eprod, bool set)
>> -{
>> - u32 swidx, sbidx, ewidx, ebidx;
>> - struct arm_smmu_ll_queue llq = {
>> - .max_n_shift = cmdq->q.llq.max_n_shift,
>> - .prod = sprod,
>> - };
>> -
>> - ewidx = BIT_WORD(Q_IDX(&llq, eprod));
>> - ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
>> -
>> - while (llq.prod != eprod) {
>> - unsigned long mask;
>> - atomic_long_t *ptr;
>> - u32 limit = BITS_PER_LONG;
>> -
>> - swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
>> - sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
>> -
>> - ptr = &cmdq->valid_map[swidx];
>> -
>> - if ((swidx == ewidx) && (sbidx < ebidx))
>> - limit = ebidx;
>> -
>> - mask = GENMASK(limit - 1, sbidx);
>> -
>> - /*
>> - * The valid bit is the inverse of the wrap bit. This means
>> - * that a zero-initialised queue is invalid and, after marking
>> - * all entries as valid, they become invalid again when we
>> - * wrap.
>> - */
>> - if (set) {
>> - atomic_long_xor(mask, ptr);
>> - } else { /* Poll */
>> - unsigned long valid;
>> + struct arm_smmu_queue *q = &smmu->cmdq.q;
>> + bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
>>
>> - valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
>> - atomic_long_cond_read_relaxed(ptr, (VAL & mask) ==
>> valid);
>> - }
>> + smmu->prev_cmd_opcode = FIELD_GET(CMDQ_0_OP, cmd[0]);
>>
>> - llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
>> + while (queue_insert_raw(q, cmd) == -ENOSPC) {
>> + if (queue_poll_cons(q, false, wfe))
>> + dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
>> }
>> }
>>
>> -/* Mark all entries in the range [sprod, eprod) as valid */
>> -static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
>> - u32 sprod, u32 eprod)
>> -{
>> - __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
>> -}
>> -
>> -/* Wait for all entries in the range [sprod, eprod) to become valid */
>> -static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
>> - u32 sprod, u32 eprod)
>> -{
>> - __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
>> -}
>> -
>> -/* Wait for the command queue to become non-full */
>> -static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
>> - struct arm_smmu_ll_queue *llq)
>> +static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
>> + struct arm_smmu_cmdq_ent *ent)
>> {
>> + u64 cmd[CMDQ_ENT_DWORDS];
>> unsigned long flags;
>> - struct arm_smmu_queue_poll qp;
>> - struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
>> - int ret = 0;
>>
>> - /*
>> - * Try to update our copy of cons by grabbing exclusive cmdq access. If
>> - * that fails, spin until somebody else updates it for us.
>> - */
>> - if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
>> - WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
>> - arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
>> - llq->val = READ_ONCE(cmdq->q.llq.val);
>> - return 0;
>> + if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
>> + dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
>> + ent->opcode);
>> + return;
>> }
>>
>> - queue_poll_init(smmu, &qp);
>> - do {
>> - llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
>> - if (!queue_full(llq))
>> - break;
>> -
>> - ret = queue_poll(&qp);
>> - } while (!ret);
>> -
>> - return ret;
>> -}
>> -
>> -/*
>> - * Wait until the SMMU signals a CMD_SYNC completion MSI.
>> - * Must be called with the cmdq lock held in some capacity.
>> - */
>> -static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
>> - struct arm_smmu_ll_queue *llq)
>> -{
>> - int ret = 0;
>> - struct arm_smmu_queue_poll qp;
>> - struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
>> - u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
>> -
>> - queue_poll_init(smmu, &qp);
>> -
>> - /*
>> - * The MSI won't generate an event, since it's being written back
>> - * into the command queue.
>> - */
>> - qp.wfe = false;
>> - smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
>> - llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
>> - return ret;
>> + spin_lock_irqsave(&smmu->cmdq.lock, flags);
>> + arm_smmu_cmdq_insert_cmd(smmu, cmd);
>> + spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
>> }
>>
>> /*
>> - * Wait until the SMMU cons index passes llq->prod.
>> - * Must be called with the cmdq lock held in some capacity.
>> + * The difference between val and sync_idx is bounded by the maximum size of
>> + * a queue at 2^20 entries, so 32 bits is plenty for wrap-safe arithmetic.
>> */
>> -static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
>> - struct arm_smmu_ll_queue *llq)
>> +static int __arm_smmu_sync_poll_msi(struct arm_smmu_device *smmu, u32
>> sync_idx)
>> {
>> - struct arm_smmu_queue_poll qp;
>> - struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
>> - u32 prod = llq->prod;
>> - int ret = 0;
>> + ktime_t timeout;
>> + u32 val;
>>
>> - queue_poll_init(smmu, &qp);
>> - llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
>> - do {
>> - if (queue_consumed(llq, prod))
>> - break;
>> -
>> - ret = queue_poll(&qp);
>> -
>> - /*
>> - * This needs to be a readl() so that our subsequent call
>> - * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
>> - *
>> - * Specifically, we need to ensure that we observe all
>> - * shared_lock()s by other CMD_SYNCs that share our owner,
>> - * so that a failing call to tryunlock() means that we're
>> - * the last one out and therefore we can safely advance
>> - * cmdq->q.llq.cons. Roughly speaking:
>> - *
>> - * CPU 0 CPU1 CPU2 (us)
>> - *
>> - * if (sync)
>> - * shared_lock();
>> - *
>> - * dma_wmb();
>> - * set_valid_map();
>> - *
>> - * if (owner) {
>> - * poll_valid_map();
>> - * <control dependency>
>> - * writel(prod_reg);
>> - *
>> - * readl(cons_reg);
>> - * tryunlock();
>> - *
>> - * Requires us to see CPU 0's shared_lock() acquisition.
>> - */
>> - llq->cons = readl(cmdq->q.cons_reg);
>> - } while (!ret);
>> + timeout = ktime_add_us(ktime_get(), ARM_SMMU_CMDQ_SYNC_TIMEOUT_US);
>> + val = smp_cond_load_acquire(&smmu->sync_count,
>> + (int)(VAL - sync_idx) >= 0 ||
>> + !ktime_before(ktime_get(), timeout));
>>
>> - return ret;
>> + return (int)(val - sync_idx) < 0 ? -ETIMEDOUT : 0;
>> }
>>
>> -static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
>> - struct arm_smmu_ll_queue *llq)
>> +static int __arm_smmu_cmdq_issue_sync_msi(struct arm_smmu_device *smmu)
>> {
>> - if (smmu->features & ARM_SMMU_FEAT_MSI &&
>> - smmu->features & ARM_SMMU_FEAT_COHERENCY)
>> - return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
>> -
>> - return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
>> -}
>> -
>> -static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64
>> *cmds,
>> - u32 prod, int n)
>> -{
>> - int i;
>> - struct arm_smmu_ll_queue llq = {
>> - .max_n_shift = cmdq->q.llq.max_n_shift,
>> - .prod = prod,
>> - };
>> -
>> - for (i = 0; i < n; ++i) {
>> - u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
>> -
>> - prod = queue_inc_prod_n(&llq, i);
>> - queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
>> - }
>> -}
>> -
>> -/*
>> - * This is the actual insertion function, and provides the following
>> - * ordering guarantees to callers:
>> - *
>> - * - There is a dma_wmb() before publishing any commands to the queue.
>> - * This can be relied upon to order prior writes to data structures
>> - * in memory (such as a CD or an STE) before the command.
>> - *
>> - * - On completion of a CMD_SYNC, there is a control dependency.
>> - * This can be relied upon to order subsequent writes to memory (e.g.
>> - * freeing an IOVA) after completion of the CMD_SYNC.
>> - *
>> - * - Command insertion is totally ordered, so if two CPUs each race to
>> - * insert their own list of commands then all of the commands from one
>> - * CPU will appear before any of the commands from the other CPU.
>> - */
>> -static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
>> - u64 *cmds, int n, bool sync)
>> -{
>> - u64 cmd_sync[CMDQ_ENT_DWORDS];
>> - u32 prod;
>> + u64 cmd[CMDQ_ENT_DWORDS];
>> unsigned long flags;
>> - bool owner;
>> - struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
>> - struct arm_smmu_ll_queue llq = {
>> - .max_n_shift = cmdq->q.llq.max_n_shift,
>> - }, head = llq;
>> - int ret = 0;
>> -
>> - /* 1. Allocate some space in the queue */
>> - local_irq_save(flags);
>> - llq.val = READ_ONCE(cmdq->q.llq.val);
>> - do {
>> - u64 old;
>> -
>> - while (!queue_has_space(&llq, n + sync)) {
>> - local_irq_restore(flags);
>> - if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
>> - dev_err_ratelimited(smmu->dev, "CMDQ
>> timeout\n");
>> - local_irq_save(flags);
>> - }
>> -
>> - head.cons = llq.cons;
>> - head.prod = queue_inc_prod_n(&llq, n + sync) |
>> - CMDQ_PROD_OWNED_FLAG;
>> -
>> - old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
>> - if (old == llq.val)
>> - break;
>> -
>> - llq.val = old;
>> - } while (1);
>> - owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
>> - head.prod &= ~CMDQ_PROD_OWNED_FLAG;
>> - llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
>> -
>> - /*
>> - * 2. Write our commands into the queue
>> - * Dependency ordering from the cmpxchg() loop above.
>> - */
>> - arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
>> - if (sync) {
>> - prod = queue_inc_prod_n(&llq, n);
>> - arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
>> - queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
>> -
>> - /*
>> - * In order to determine completion of our CMD_SYNC, we must
>> - * ensure that the queue can't wrap twice without us noticing.
>> - * We achieve that by taking the cmdq lock as shared before
>> - * marking our slot as valid.
>> - */
>> - arm_smmu_cmdq_shared_lock(cmdq);
>> - }
>> -
>> - /* 3. Mark our slots as valid, ensuring commands are visible first */
>> - dma_wmb();
>> - arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
>> -
>> - /* 4. If we are the owner, take control of the SMMU hardware */
>> - if (owner) {
>> - /* a. Wait for previous owner to finish */
>> - atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
>> -
>> - /* b. Stop gathering work by clearing the owned flag */
>> - prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
>> - &cmdq->q.llq.atomic.prod);
>> - prod &= ~CMDQ_PROD_OWNED_FLAG;
>> + struct arm_smmu_cmdq_ent ent = {
>> + .opcode = CMDQ_OP_CMD_SYNC,
>> + .sync = {
>> + .msiaddr = virt_to_phys(&smmu->sync_count),
>> + },
>> + };
>>
>> - /*
>> - * c. Wait for any gathered work to be written to the queue.
>> - * Note that we read our own entries so that we have the control
>> - * dependency required by (d).
>> - */
>> - arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
>> + spin_lock_irqsave(&smmu->cmdq.lock, flags);
>>
>> - /*
>> - * d. Advance the hardware prod pointer
>> - * Control dependency ordering from the entries becoming valid.
>> - */
>> - writel_relaxed(prod, cmdq->q.prod_reg);
>> -
>> - /*
>> - * e. Tell the next owner we're done
>> - * Make sure we've updated the hardware first, so that we don't
>> - * race to update prod and potentially move it backwards.
>> - */
>> - atomic_set_release(&cmdq->owner_prod, prod);
>> + /* Piggy-back on the previous command if it's a SYNC */
>> + if (smmu->prev_cmd_opcode == CMDQ_OP_CMD_SYNC) {
>> + ent.sync.msidata = smmu->sync_nr;
>> + } else {
>> + ent.sync.msidata = ++smmu->sync_nr;
>> + arm_smmu_cmdq_build_cmd(cmd, &ent);
>> + arm_smmu_cmdq_insert_cmd(smmu, cmd);
>> }
>>
>> - /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
>> - if (sync) {
>> - llq.prod = queue_inc_prod_n(&llq, n);
>> - ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
>> - if (ret) {
>> - dev_err_ratelimited(smmu->dev,
>> - "CMD_SYNC timeout at 0x%08x [hwprod
>> 0x%08x, hwcons 0x%08x]\n",
>> - llq.prod,
>> - readl_relaxed(cmdq->q.prod_reg),
>> - readl_relaxed(cmdq->q.cons_reg));
>> - }
>> -
>> - /*
>> - * Try to unlock the cmdq lock. This will fail if we're the last
>> - * reader, in which case we can safely update cmdq->q.llq.cons
>> - */
>> - if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
>> - WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
>> - arm_smmu_cmdq_shared_unlock(cmdq);
>> - }
>> - }
>> + spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
>>
>> - local_irq_restore(flags);
>> - return ret;
>> + return __arm_smmu_sync_poll_msi(smmu, ent.sync.msidata);
>> }
>>
>> -static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
>> - struct arm_smmu_cmdq_ent *ent)
>> +static int __arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
>> {
>> u64 cmd[CMDQ_ENT_DWORDS];
>> + unsigned long flags;
>> + bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
>> + struct arm_smmu_cmdq_ent ent = { .opcode = CMDQ_OP_CMD_SYNC };
>> + int ret;
>>
>> - if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
>> - dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
>> - ent->opcode);
>> - return -EINVAL;
>> - }
>> + arm_smmu_cmdq_build_cmd(cmd, &ent);
>>
>> - return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
>> -}
>> + spin_lock_irqsave(&smmu->cmdq.lock, flags);
>> + arm_smmu_cmdq_insert_cmd(smmu, cmd);
>> + ret = queue_poll_cons(&smmu->cmdq.q, true, wfe);
>> + spin_unlock_irqrestore(&smmu->cmdq.lock, flags);
>>
>> -static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
>> -{
>> - return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
>> + return ret;
>> }
>>
>> -static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
>> - struct arm_smmu_cmdq_batch *cmds,
>> - struct arm_smmu_cmdq_ent *cmd)
>> +static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
>> {
>> - if (cmds->num == CMDQ_BATCH_ENTRIES) {
>> - arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);
>> - cmds->num = 0;
>> - }
>> - arm_smmu_cmdq_build_cmd(&cmds->cmds[cmds->num * CMDQ_ENT_DWORDS], cmd);
>> - cmds->num++;
>> -}
>> + int ret;
>> + bool msi = (smmu->features & ARM_SMMU_FEAT_MSI) &&
>> + (smmu->features & ARM_SMMU_FEAT_COHERENCY);
>>
>> -static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
>> - struct arm_smmu_cmdq_batch *cmds)
>> -{
>> - return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
>> + ret = msi ? __arm_smmu_cmdq_issue_sync_msi(smmu)
>> + : __arm_smmu_cmdq_issue_sync(smmu);
>> + if (ret)
>> + dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
>> + return ret;
>> }
>>
>> /* Context descriptor manipulation functions */
>> @@ -1535,7 +1137,6 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain
>> *smmu_domain,
>> size_t i;
>> unsigned long flags;
>> struct arm_smmu_master *master;
>> - struct arm_smmu_cmdq_batch cmds = {};
>> struct arm_smmu_device *smmu = smmu_domain->smmu;
>> struct arm_smmu_cmdq_ent cmd = {
>> .opcode = CMDQ_OP_CFGI_CD,
>> @@ -1549,12 +1150,12 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain
>> *smmu_domain,
>> list_for_each_entry(master, &smmu_domain->devices, domain_head) {
>> for (i = 0; i < master->num_sids; i++) {
>> cmd.cfgi.sid = master->sids[i];
>> - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
>> + arm_smmu_cmdq_issue_cmd(smmu, &cmd);
>> }
>> }
>> spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
>>
>> - arm_smmu_cmdq_batch_submit(smmu, &cmds);
>> + arm_smmu_cmdq_issue_sync(smmu);
>> }
>>
>> static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
>> @@ -2189,16 +1790,17 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long
>> iova, size_t size,
>> cmd->atc.size = log2_span;
>> }
>>
>> -static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
>> +static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
>> + struct arm_smmu_cmdq_ent *cmd)
>> {
>> int i;
>> - struct arm_smmu_cmdq_ent cmd;
>>
>> - arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
>> + if (!master->ats_enabled)
>> + return 0;
>>
>> for (i = 0; i < master->num_sids; i++) {
>> - cmd.atc.sid = master->sids[i];
>> - arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
>> + cmd->atc.sid = master->sids[i];
>> + arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
>> }
>>
>> return arm_smmu_cmdq_issue_sync(master->smmu);
>> @@ -2207,11 +1809,10 @@ static int arm_smmu_atc_inv_master(struct
>> arm_smmu_master *master)
>> static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
>> int ssid, unsigned long iova, size_t size)
>> {
>> - int i;
>> + int ret = 0;
>> unsigned long flags;
>> struct arm_smmu_cmdq_ent cmd;
>> struct arm_smmu_master *master;
>> - struct arm_smmu_cmdq_batch cmds = {};
>>
>> if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
>> return 0;
>> @@ -2236,18 +1837,11 @@ static int arm_smmu_atc_inv_domain(struct
>> arm_smmu_domain *smmu_domain,
>> arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
>>
>> spin_lock_irqsave(&smmu_domain->devices_lock, flags);
>> - list_for_each_entry(master, &smmu_domain->devices, domain_head) {
>> - if (!master->ats_enabled)
>> - continue;
>> -
>> - for (i = 0; i < master->num_sids; i++) {
>> - cmd.atc.sid = master->sids[i];
>> - arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
>> - }
>> - }
>> + list_for_each_entry(master, &smmu_domain->devices, domain_head)
>> + ret |= arm_smmu_atc_inv_master(master, &cmd);
>> spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
>>
>> - return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
>> + return ret ? -ETIMEDOUT : 0;
>> }
>>
>> /* IO_PGTABLE API */
>> @@ -2269,32 +1863,27 @@ static void arm_smmu_tlb_inv_context(void *cookie)
>> /*
>> * NOTE: when io-pgtable is in non-strict mode, we may get here with
>> * PTEs previously cleared by unmaps on the current CPU not yet visible
>> - * to the SMMU. We are relying on the dma_wmb() implicit during cmd
>> - * insertion to guarantee those are observed before the TLBI. Do be
>> - * careful, 007.
>> + * to the SMMU. We are relying on the DSB implicit in
>> + * queue_sync_prod_out() to guarantee those are observed before the
>> + * TLBI. Do be careful, 007.
>> */
>> arm_smmu_cmdq_issue_cmd(smmu, &cmd);
>> arm_smmu_cmdq_issue_sync(smmu);
>> arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
>> }
>>
>> -static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
>> - size_t granule, bool leaf,
>> - struct arm_smmu_domain *smmu_domain)
>> +static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
>> + size_t granule, bool leaf, void
>> *cookie)
>> {
>> + struct arm_smmu_domain *smmu_domain = cookie;
>> struct arm_smmu_device *smmu = smmu_domain->smmu;
>> - unsigned long start = iova, end = iova + size, num_pages = 0, tg = 0;
>> - size_t inv_range = granule;
>> - struct arm_smmu_cmdq_batch cmds = {};
>> struct arm_smmu_cmdq_ent cmd = {
>> .tlbi = {
>> .leaf = leaf,
>> + .addr = iova,
>> },
>> };
>>
>> - if (!size)
>> - return;
>> -
>> if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
>> cmd.opcode = CMDQ_OP_TLBI_NH_VA;
>> cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
>> @@ -2303,78 +1892,37 @@ static void arm_smmu_tlb_inv_range(unsigned long
>> iova, size_t size,
>> cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
>> }
>>
>> - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
>> - /* Get the leaf page size */
>> - tg = __ffs(smmu_domain->domain.pgsize_bitmap);
>> -
>> - /* Convert page size of 12,14,16 (log2) to 1,2,3 */
>> - cmd.tlbi.tg = (tg - 10) / 2;
>> -
>> - /* Determine what level the granule is at */
>> - cmd.tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
>> -
>> - num_pages = size >> tg;
>> - }
>> -
>> - while (iova < end) {
>> - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
>> - /*
>> - * On each iteration of the loop, the range is 5 bits
>> - * worth of the aligned size remaining.
>> - * The range in pages is:
>> - *
>> - * range = (num_pages & (0x1f << __ffs(num_pages)))
>> - */
>> - unsigned long scale, num;
>> -
>> - /* Determine the power of 2 multiple number of pages */
>> - scale = __ffs(num_pages);
>> - cmd.tlbi.scale = scale;
>> -
>> - /* Determine how many chunks of 2^scale size we have */
>> - num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
>> - cmd.tlbi.num = num - 1;
>> -
>> - /* range is num * 2^scale * pgsize */
>> - inv_range = num << (scale + tg);
>> -
>> - /* Clear out the lower order bits for the next
>> iteration */
>> - num_pages -= num << scale;
>> - }
>> -
>> - cmd.tlbi.addr = iova;
>> - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
>> - iova += inv_range;
>> - }
>> - arm_smmu_cmdq_batch_submit(smmu, &cmds);
>> -
>> - /*
>> - * Unfortunately, this can't be leaf-only since we may have
>> - * zapped an entire table.
>> - */
>> - arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);
>> + do {
>> + arm_smmu_cmdq_issue_cmd(smmu, &cmd);
>> + cmd.tlbi.addr += granule;
>> + } while (size -= granule);
>> }
>>
>> static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
>> unsigned long iova, size_t granule,
>> void *cookie)
>> {
>> - struct arm_smmu_domain *smmu_domain = cookie;
>> - struct iommu_domain *domain = &smmu_domain->domain;
>> -
>> - iommu_iotlb_gather_add_page(domain, gather, iova, granule);
>> + arm_smmu_tlb_inv_range_nosync(iova, granule, granule, true, cookie);
>> }
>>
>> static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
>> size_t granule, void *cookie)
>> {
>> - arm_smmu_tlb_inv_range(iova, size, granule, false, cookie);
>> + struct arm_smmu_domain *smmu_domain = cookie;
>> + struct arm_smmu_device *smmu = smmu_domain->smmu;
>> +
>> + arm_smmu_tlb_inv_range_nosync(iova, size, granule, false, cookie);
>> + arm_smmu_cmdq_issue_sync(smmu);
>> }
>>
>> static void arm_smmu_tlb_inv_leaf(unsigned long iova, size_t size,
>> size_t granule, void *cookie)
>> {
>> - arm_smmu_tlb_inv_range(iova, size, granule, true, cookie);
>> + struct arm_smmu_domain *smmu_domain = cookie;
>> + struct arm_smmu_device *smmu = smmu_domain->smmu;
>> +
>> + arm_smmu_tlb_inv_range_nosync(iova, size, granule, true, cookie);
>> + arm_smmu_cmdq_issue_sync(smmu);
>> }
>>
>> static const struct iommu_flush_ops arm_smmu_flush_ops = {
>> @@ -2700,6 +2248,7 @@ static void arm_smmu_enable_ats(struct arm_smmu_master
>> *master)
>>
>> static void arm_smmu_disable_ats(struct arm_smmu_master *master)
>> {
>> + struct arm_smmu_cmdq_ent cmd;
>> struct arm_smmu_domain *smmu_domain = master->domain;
>>
>> if (!master->ats_enabled)
>> @@ -2711,8 +2260,9 @@ static void arm_smmu_disable_ats(struct
>> arm_smmu_master *master)
>> * ATC invalidation via the SMMU.
>> */
>> wmb();
>> - arm_smmu_atc_inv_master(master);
>> - atomic_dec(&smmu_domain->nr_ats_masters);
>> + arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
>> + arm_smmu_atc_inv_master(master, &cmd);
>> + atomic_dec(&smmu_domain->nr_ats_masters);
>> }
>>
>> static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
>> @@ -2875,10 +2425,10 @@ static void arm_smmu_flush_iotlb_all(struct
>> iommu_domain *domain)
>> static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
>> struct iommu_iotlb_gather *gather)
>> {
>> - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
>> + struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
>>
>> - arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start,
>> - gather->pgsize, true, smmu_domain);
>> + if (smmu)
>> + arm_smmu_cmdq_issue_sync(smmu);
>> }
>>
>> static phys_addr_t
>> @@ -3176,49 +2726,18 @@ static int arm_smmu_init_one_queue(struct
>> arm_smmu_device *smmu,
>> return 0;
>> }
>>
>> -static void arm_smmu_cmdq_free_bitmap(void *data)
>> -{
>> - unsigned long *bitmap = data;
>> - bitmap_free(bitmap);
>> -}
>> -
>> -static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
>> -{
>> - int ret = 0;
>> - struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
>> - unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
>> - atomic_long_t *bitmap;
>> -
>> - atomic_set(&cmdq->owner_prod, 0);
>> - atomic_set(&cmdq->lock, 0);
>> -
>> - bitmap = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
>> - if (!bitmap) {
>> - dev_err(smmu->dev, "failed to allocate cmdq bitmap\n");
>> - ret = -ENOMEM;
>> - } else {
>> - cmdq->valid_map = bitmap;
>> - devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap);
>> - }
>> -
>> - return ret;
>> -}
>> -
>> static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
>> {
>> int ret;
>>
>> /* cmdq */
>> + spin_lock_init(&smmu->cmdq.lock);
>> ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
>> ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS,
>> "cmdq");
>> if (ret)
>> return ret;
>>
>> - ret = arm_smmu_cmdq_init(smmu);
>> - if (ret)
>> - return ret;
>> -
>> /* evtq */
>> ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
>> ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS,
>> @@ -3799,15 +3318,9 @@ static int arm_smmu_device_hw_probe(struct
>> arm_smmu_device *smmu)
>> /* Queue sizes, capped to ensure natural alignment */
>> smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
>> FIELD_GET(IDR1_CMDQS, reg));
>> - if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
>> - /*
>> - * We don't support splitting up batches, so one batch of
>> - * commands plus an extra sync needs to fit inside the command
>> - * queue. There's also no way we can handle the weird alignment
>> - * restrictions on the base pointer for a unit-length queue.
>> - */
>> - dev_err(smmu->dev, "command queue size <= %d entries not
>> supported\n",
>> - CMDQ_BATCH_ENTRIES);
>> + if (!smmu->cmdq.q.llq.max_n_shift) {
>> + /* Odd alignment restrictions on the base, so ignore for now */
>> + dev_err(smmu->dev, "unit-length command queue not supported\n");
>> return -ENXIO;
>> }
>>
>> --
>> 2.17.1
>>
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |