From 97a644208d1a08b7104d1fe2ace8cef011222711 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Fri, 3 Jan 2014 12:01:26 +0000 Subject: [PATCH 1/6] iommu/arm-smmu: fix pud/pmd entry fill sequence The ARM SMMU driver's population of puds and pmds is broken, since we iterate over the next level of table repeatedly setting the current level descriptor to point at the pmd being initialised. This is clearly wrong when dealing with multiple pmds/puds. This patch fixes the problem by moving the pud/pmd population out of the loop and instead performing it when we allocate the next level (like we correctly do for ptes already). The starting address for the next level is then calculated prior to entering the loop. Cc: Signed-off-by: Yifan Zhang Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 8911850c9444..9f210de6537e 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1320,6 +1320,11 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud, pmd = pmd_alloc_one(NULL, addr); if (!pmd) return -ENOMEM; + + pud_populate(NULL, pud, pmd); + arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud)); + + pmd += pmd_index(addr); } else #endif pmd = pmd_offset(pud, addr); @@ -1328,8 +1333,6 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud, next = pmd_addr_end(addr, end); ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, end, pfn, flags, stage); - pud_populate(NULL, pud, pmd); - arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud)); phys += next - addr; } while (pmd++, addr = next, addr < end); @@ -1349,6 +1352,11 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd, pud = pud_alloc_one(NULL, addr); if (!pud) return -ENOMEM; + + pgd_populate(NULL, pgd, pud); + arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd)); + + pud += pud_index(addr); } else #endif pud = pud_offset(pgd, addr); @@ -1357,8 +1365,6 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd, next = pud_addr_end(addr, end); ret = arm_smmu_alloc_init_pmd(smmu, pud, addr, next, phys, flags, stage); - pgd_populate(NULL, pud, pgd); - arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd)); phys += next - addr; } while (pud++, addr = next, addr < end); From c9d09e2748eaa55cac2af274574baa6368189bc1 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 4 Feb 2014 22:12:42 +0000 Subject: [PATCH 2/6] iommu/arm-smmu: really fix page table locking Commit a44a9791e778 ("iommu/arm-smmu: use mutex instead of spinlock for locking page tables") replaced the page table spinlock with a mutex, to allow blocking allocations to satisfy lazy mapping requests. Unfortunately, it turns out that IOMMU mappings are created from atomic context (e.g. spinlock held during a dma_map), so this change doesn't really help us in practice. This patch is a partial revert of the offending commit, bringing back the original spinlock but replacing our page table allocations for any levels below the pgd (which is allocated during domain init) with GFP_ATOMIC instead of GFP_KERNEL. Cc: Reported-by: Andreas Herrmann Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 9f210de6537e..6eb54ae97470 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -393,7 +393,7 @@ struct arm_smmu_domain { struct arm_smmu_cfg root_cfg; phys_addr_t output_mask; - struct mutex lock; + spinlock_t lock; }; static DEFINE_SPINLOCK(arm_smmu_devices_lock); @@ -901,7 +901,7 @@ static int arm_smmu_domain_init(struct iommu_domain *domain) goto out_free_domain; smmu_domain->root_cfg.pgd = pgd; - mutex_init(&smmu_domain->lock); + spin_lock_init(&smmu_domain->lock); domain->priv = smmu_domain; return 0; @@ -1138,7 +1138,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) * Sanity check the domain. We don't currently support domains * that cross between different SMMU chains. */ - mutex_lock(&smmu_domain->lock); + spin_lock(&smmu_domain->lock); if (!smmu_domain->leaf_smmu) { /* Now that we have a master, we can finalise the domain */ ret = arm_smmu_init_domain_context(domain, dev); @@ -1153,7 +1153,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) dev_name(device_smmu->dev)); goto err_unlock; } - mutex_unlock(&smmu_domain->lock); + spin_unlock(&smmu_domain->lock); /* Looks ok, so add the device to the domain */ master = find_smmu_master(smmu_domain->leaf_smmu, dev->of_node); @@ -1163,7 +1163,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return arm_smmu_domain_add_master(smmu_domain, master); err_unlock: - mutex_unlock(&smmu_domain->lock); + spin_unlock(&smmu_domain->lock); return ret; } @@ -1210,7 +1210,7 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd, if (pmd_none(*pmd)) { /* Allocate a new set of tables */ - pgtable_t table = alloc_page(PGALLOC_GFP); + pgtable_t table = alloc_page(GFP_ATOMIC|__GFP_ZERO); if (!table) return -ENOMEM; @@ -1317,7 +1317,7 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud, #ifndef __PAGETABLE_PMD_FOLDED if (pud_none(*pud)) { - pmd = pmd_alloc_one(NULL, addr); + pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); if (!pmd) return -ENOMEM; @@ -1349,7 +1349,7 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd, #ifndef __PAGETABLE_PUD_FOLDED if (pgd_none(*pgd)) { - pud = pud_alloc_one(NULL, addr); + pud = (pud_t *)get_zeroed_page(GFP_ATOMIC); if (!pud) return -ENOMEM; @@ -1403,7 +1403,7 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, if (paddr & ~output_mask) return -ERANGE; - mutex_lock(&smmu_domain->lock); + spin_lock(&smmu_domain->lock); pgd += pgd_index(iova); end = iova + size; do { @@ -1419,7 +1419,7 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, } while (pgd++, iova != end); out_unlock: - mutex_unlock(&smmu_domain->lock); + spin_unlock(&smmu_domain->lock); /* Ensure new page tables are visible to the hardware walker */ if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) From 6dd35f45b8dac827b6f9dd86f5aca6436cdd2410 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 5 Feb 2014 17:49:34 +0000 Subject: [PATCH 3/6] iommu/arm-smmu: fix table flushing during initial allocations Now that we populate page tables as we traverse them ("iommu/arm-smmu: fix pud/pmd entry fill sequence"), we need to ensure that we flush out our zeroed tables after initial allocation, to prevent speculative TLB fills using bogus data. This patch adds additional calls to arm_smmu_flush_pgtable during initial table allocation, and moves the dsb required by coherent table walkers into the helper. Cc: Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 51 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 6eb54ae97470..509f01f054d9 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -79,7 +79,6 @@ #define ARM_SMMU_PTE_CONT_SIZE (PAGE_SIZE * ARM_SMMU_PTE_CONT_ENTRIES) #define ARM_SMMU_PTE_CONT_MASK (~(ARM_SMMU_PTE_CONT_SIZE - 1)) -#define ARM_SMMU_PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(pte_t)) /* Stage-1 PTE */ #define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6) @@ -632,6 +631,28 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) return IRQ_HANDLED; } +static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr, + size_t size) +{ + unsigned long offset = (unsigned long)addr & ~PAGE_MASK; + + + /* Ensure new page tables are visible to the hardware walker */ + if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) { + dsb(); + } else { + /* + * If the SMMU can't walk tables in the CPU caches, treat them + * like non-coherent DMA since we need to flush the new entries + * all the way out to memory. There's no possibility of + * recursion here as the SMMU table walker will not be wired + * through another SMMU. + */ + dma_map_page(smmu->dev, virt_to_page(addr), offset, size, + DMA_TO_DEVICE); + } +} + static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) { u32 reg; @@ -715,6 +736,8 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) } /* TTBR0 */ + arm_smmu_flush_pgtable(smmu, root_cfg->pgd, + PTRS_PER_PGD * sizeof(pgd_t)); reg = __pa(root_cfg->pgd); writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32; @@ -1177,23 +1200,6 @@ static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_domain_remove_master(smmu_domain, master); } -static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr, - size_t size) -{ - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; - - /* - * If the SMMU can't walk tables in the CPU caches, treat them - * like non-coherent DMA since we need to flush the new entries - * all the way out to memory. There's no possibility of recursion - * here as the SMMU table walker will not be wired through another - * SMMU. - */ - if (!(smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)) - dma_map_page(smmu->dev, virt_to_page(addr), offset, size, - DMA_TO_DEVICE); -} - static bool arm_smmu_pte_is_contiguous_range(unsigned long addr, unsigned long end) { @@ -1214,8 +1220,7 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd, if (!table) return -ENOMEM; - arm_smmu_flush_pgtable(smmu, page_address(table), - ARM_SMMU_PTE_HWTABLE_SIZE); + arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE); if (!pgtable_page_ctor(table)) { __free_page(table); return -ENOMEM; @@ -1321,6 +1326,7 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud, if (!pmd) return -ENOMEM; + arm_smmu_flush_pgtable(smmu, pmd, PAGE_SIZE); pud_populate(NULL, pud, pmd); arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud)); @@ -1353,6 +1359,7 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd, if (!pud) return -ENOMEM; + arm_smmu_flush_pgtable(smmu, pud, PAGE_SIZE); pgd_populate(NULL, pgd, pud); arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd)); @@ -1421,10 +1428,6 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, out_unlock: spin_unlock(&smmu_domain->lock); - /* Ensure new page tables are visible to the hardware walker */ - if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) - dsb(); - return ret; } From 57ca90f6800987ac274d7ba065ae6692cdf9bcd7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 6 Feb 2014 14:59:05 +0000 Subject: [PATCH 4/6] iommu/arm-smmu: set CBARn.BPSHCFG to NSH for s1-s2-bypass contexts Whilst trying to bring-up an SMMUv2 implementation with the table walker plumbed into a coherent interconnect, I noticed that the memory transactions targetting the CPU caches from the SMMU were marked as outer-shareable instead of inner-shareable. After a bunch of digging, it seems that we actually need to program CBARn.BPSHCFG for s1-s2-bypass contexts to act as non-shareable in order for the shareability configured in the corresponding TTBCR not to be overridden with an outer-shareable attribute. Cc: Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 509f01f054d9..0ae4dd39197f 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -190,6 +190,9 @@ #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) #define CBAR_VMID_SHIFT 0 #define CBAR_VMID_MASK 0xff +#define CBAR_S1_BPSHCFG_SHIFT 8 +#define CBAR_S1_BPSHCFG_MASK 3 +#define CBAR_S1_BPSHCFG_NSH 3 #define CBAR_S1_MEMATTR_SHIFT 12 #define CBAR_S1_MEMATTR_MASK 0xf #define CBAR_S1_MEMATTR_WB 0xf @@ -671,11 +674,16 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) if (smmu->version == 1) reg |= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT; - /* Use the weakest memory type, so it is overridden by the pte */ - if (stage1) - reg |= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); - else + /* + * Use the weakest shareability/memory types, so they are + * overridden by the ttbcr/pte. + */ + if (stage1) { + reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) | + (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); + } else { reg |= ARM_SMMU_CB_VMID(root_cfg) << CBAR_VMID_SHIFT; + } writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx)); if (smmu->version > 1) { From d123cf82d339c5cc4ffe2a481e0caa23a501d4ac Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 4 Feb 2014 22:17:53 +0000 Subject: [PATCH 5/6] iommu/arm-smmu: fix compilation issue when !CONFIG_ARM_AMBA If !CONFIG_ARM_AMBA, we shouldn't try to register ourselves with the amba_bustype. Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 0ae4dd39197f..6fe7922ecc1d 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -2004,8 +2004,10 @@ static int __init arm_smmu_init(void) if (!iommu_present(&platform_bus_type)) bus_set_iommu(&platform_bus_type, &arm_smmu_ops); +#ifdef CONFIG_ARM_AMBA if (!iommu_present(&amba_bustype)) bus_set_iommu(&amba_bustype, &arm_smmu_ops); +#endif return 0; } From 972157cac528f6cfd1f7e640139287951066106e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 20 Feb 2014 12:19:08 +0100 Subject: [PATCH 6/6] arm/smmu: Use irqsafe spinlock for domain lock As the lock might be used through DMA-API which is allowed in interrupt context. Signed-off-by: Joerg Roedel Acked-by: Will Deacon --- drivers/iommu/arm-smmu.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 6fe7922ecc1d..1d9ab39af29f 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1159,6 +1159,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) struct arm_smmu_domain *smmu_domain = domain->priv; struct arm_smmu_device *device_smmu = dev->archdata.iommu; struct arm_smmu_master *master; + unsigned long flags; if (!device_smmu) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); @@ -1169,7 +1170,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) * Sanity check the domain. We don't currently support domains * that cross between different SMMU chains. */ - spin_lock(&smmu_domain->lock); + spin_lock_irqsave(&smmu_domain->lock, flags); if (!smmu_domain->leaf_smmu) { /* Now that we have a master, we can finalise the domain */ ret = arm_smmu_init_domain_context(domain, dev); @@ -1184,7 +1185,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) dev_name(device_smmu->dev)); goto err_unlock; } - spin_unlock(&smmu_domain->lock); + spin_unlock_irqrestore(&smmu_domain->lock, flags); /* Looks ok, so add the device to the domain */ master = find_smmu_master(smmu_domain->leaf_smmu, dev->of_node); @@ -1194,7 +1195,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return arm_smmu_domain_add_master(smmu_domain, master); err_unlock: - spin_unlock(&smmu_domain->lock); + spin_unlock_irqrestore(&smmu_domain->lock, flags); return ret; } @@ -1396,6 +1397,7 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; pgd_t *pgd = root_cfg->pgd; struct arm_smmu_device *smmu = root_cfg->smmu; + unsigned long irqflags; if (root_cfg->cbar == CBAR_TYPE_S2_TRANS) { stage = 2; @@ -1418,7 +1420,7 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, if (paddr & ~output_mask) return -ERANGE; - spin_lock(&smmu_domain->lock); + spin_lock_irqsave(&smmu_domain->lock, irqflags); pgd += pgd_index(iova); end = iova + size; do { @@ -1434,7 +1436,7 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, } while (pgd++, iova != end); out_unlock: - spin_unlock(&smmu_domain->lock); + spin_unlock_irqrestore(&smmu_domain->lock, irqflags); return ret; }