From b6d07e0273d3296cfbdc88145b8a00ddbefb310a Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 18 Apr 2014 14:19:50 +0200 Subject: [PATCH 1/4] PCI: mvebu: fix off-by-one in the computed size of the mbus windows mvebu_pcie_handle_membase_change() and mvebu_pcie_handle_iobase_change() do not correctly compute the window size. PCI uses an inclusive start/end address pair, which requires a +1 when converting to size. This only worked because a bug in the mbus driver allowed it to silently accept and round up bogus sizes. Fix this by adding one to the computed size. Fixes: 45361a4fe446 ('PCIe driver for Marvell Armada 370/XP systems') Cc: # v3.11+ Signed-off-by: Willy Tarreau Reviewed-By: Jason Gunthorpe Signed-off-by: Thomas Petazzoni Link: https://lkml.kernel.org/r/1397823593-1932-5-git-send-email-thomas.petazzoni@free-electrons.com Tested-by: Neil Greatorex Acked-by: Bjorn Helgaas Signed-off-by: Jason Cooper --- drivers/pci/host/pci-mvebu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/host/pci-mvebu.c b/drivers/pci/host/pci-mvebu.c index d3d1cfd51e09..48299215cc59 100644 --- a/drivers/pci/host/pci-mvebu.c +++ b/drivers/pci/host/pci-mvebu.c @@ -331,7 +331,7 @@ static void mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port) port->iowin_base = port->pcie->io.start + iobase; port->iowin_size = ((0xFFF | ((port->bridge.iolimit & 0xF0) << 8) | (port->bridge.iolimitupper << 16)) - - iobase); + iobase) + 1; mvebu_mbus_add_window_remap_by_id(port->io_target, port->io_attr, port->iowin_base, port->iowin_size, @@ -364,7 +364,7 @@ static void mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port) port->memwin_base = ((port->bridge.membase & 0xFFF0) << 16); port->memwin_size = (((port->bridge.memlimit & 0xFFF0) << 16) | 0xFFFFF) - - port->memwin_base; + port->memwin_base + 1; mvebu_mbus_add_window_by_id(port->mem_target, port->mem_attr, port->memwin_base, port->memwin_size); From 09752a12f430f58523fb6f435f5e30e4048fcfb2 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 18 Apr 2014 14:19:51 +0200 Subject: [PATCH 2/4] bus: mvebu-mbus: Avoid setting an undefined window size The mbus hardware requires a power of two size, and size aligned base. Currently, if a non-power of two is passed in to the low level routines they configure the register in a way that results in undefined behaviour. Call WARN and return EINVAL instead. Also, update the debugfs routines to show a message if there is an invalid register setting. All together this makes the recent problems with silent failure of PCI very obvious, noisy and debuggable. Signed-off-by: Jason Gunthorpe Signed-off-by: Thomas Petazzoni Link: https://lkml.kernel.org/r/1397823593-1932-6-git-send-email-thomas.petazzoni@free-electrons.com Signed-off-by: Jason Cooper --- drivers/bus/mvebu-mbus.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/bus/mvebu-mbus.c b/drivers/bus/mvebu-mbus.c index 293e2e0a0a87..afee0f731111 100644 --- a/drivers/bus/mvebu-mbus.c +++ b/drivers/bus/mvebu-mbus.c @@ -56,6 +56,7 @@ #include #include #include +#include /* * DDR target is the same on all platforms. @@ -266,6 +267,17 @@ static int mvebu_mbus_setup_window(struct mvebu_mbus_state *mbus, mbus->soc->win_cfg_offset(win); u32 ctrl, remap_addr; + if (!is_power_of_2(size)) { + WARN(true, "Invalid MBus window size: 0x%zx\n", size); + return -EINVAL; + } + + if ((base & (phys_addr_t)(size - 1)) != 0) { + WARN(true, "Invalid MBus base/size: %pa len 0x%zx\n", &base, + size); + return -EINVAL; + } + ctrl = ((size - 1) & WIN_CTRL_SIZE_MASK) | (attr << WIN_CTRL_ATTR_SHIFT) | (target << WIN_CTRL_TGT_SHIFT) | @@ -413,6 +425,10 @@ static int mvebu_devs_debug_show(struct seq_file *seq, void *v) win, (unsigned long long)wbase, (unsigned long long)(wbase + wsize), wtarget, wattr); + if (!is_power_of_2(wsize) || + ((wbase & (u64)(wsize - 1)) != 0)) + seq_puts(seq, " (Invalid base/size!!)"); + if (win < mbus->soc->num_remappable_wins) { seq_printf(seq, " (remap %016llx)\n", (unsigned long long)wremap); From b566e782be32145664d96ada3e389f17d32742e5 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Fri, 18 Apr 2014 14:19:52 +0200 Subject: [PATCH 3/4] bus: mvebu-mbus: allow several windows with the same target/attribute Having multiple windows with the same target and attribute is actually legal, and can be useful for PCIe windows, when PCIe BARs have a size that isn't a power of two, and we therefore need to create several MBus windows to cover the PCIe BAR for a given PCIe interface. Fixes: fddddb52a6c4 ('bus: introduce an Marvell EBU MBus driver') Cc: # v3.10+ Signed-off-by: Thomas Petazzoni Link: https://lkml.kernel.org/r/1397823593-1932-7-git-send-email-thomas.petazzoni@free-electrons.com Tested-by: Neil Greatorex Signed-off-by: Jason Cooper --- drivers/bus/mvebu-mbus.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/bus/mvebu-mbus.c b/drivers/bus/mvebu-mbus.c index afee0f731111..00b73448b22e 100644 --- a/drivers/bus/mvebu-mbus.c +++ b/drivers/bus/mvebu-mbus.c @@ -223,12 +223,6 @@ static int mvebu_mbus_window_conflicts(struct mvebu_mbus_state *mbus, */ if ((u64)base < wend && end > wbase) return 0; - - /* - * Check if target/attribute conflicts - */ - if (target == wtarget && attr == wattr) - return 0; } return 1; From 398f5d5e10b6b917cd9d35ef21d545b0afbada22 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Fri, 18 Apr 2014 14:19:53 +0200 Subject: [PATCH 4/4] PCI: mvebu: split PCIe BARs into multiple MBus windows when needed MBus windows are used on Marvell platforms to map certain peripherals in the physical address space. In the PCIe context, MBus windows are needed to map PCIe I/O and memory regions in the physical address. However, those MBus windows can only have power of two sizes, while PCIe BAR do not necessarily guarantee this. For this reason, the current pci-mvebu breaks on platforms where PCIe devices have BARs that don't sum up to a power of two size at the emulated bridge level. This commit fixes this by allowing the pci-mvebu driver to create multiple contiguous MBus windows (each having a power of two size) to cover a given PCIe BAR. To achieve this, two functions are added: mvebu_pcie_add_windows() and mvebu_pcie_del_windows() to respectively add and remove all the MBus windows that are needed to map the provided PCIe region base and size. The emulated PCI bridge code now calls those functions, instead of directly calling the mvebu-mbus driver functions. Fixes: 45361a4fe446 ('pci: PCIe driver for Marvell Armada 370/XP systems') Cc: # v3.11+ Signed-off-by: Thomas Petazzoni Link: https://lkml.kernel.org/r/1397823593-1932-8-git-send-email-thomas.petazzoni@free-electrons.com Tested-by: Neil Greatorex Acked-by: Bjorn Helgaas Signed-off-by: Jason Cooper --- drivers/pci/host/pci-mvebu.c | 88 ++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/drivers/pci/host/pci-mvebu.c b/drivers/pci/host/pci-mvebu.c index 48299215cc59..e384e2534594 100644 --- a/drivers/pci/host/pci-mvebu.c +++ b/drivers/pci/host/pci-mvebu.c @@ -293,6 +293,58 @@ static int mvebu_pcie_hw_wr_conf(struct mvebu_pcie_port *port, return PCIBIOS_SUCCESSFUL; } +/* + * Remove windows, starting from the largest ones to the smallest + * ones. + */ +static void mvebu_pcie_del_windows(struct mvebu_pcie_port *port, + phys_addr_t base, size_t size) +{ + while (size) { + size_t sz = 1 << (fls(size) - 1); + + mvebu_mbus_del_window(base, sz); + base += sz; + size -= sz; + } +} + +/* + * MBus windows can only have a power of two size, but PCI BARs do not + * have this constraint. Therefore, we have to split the PCI BAR into + * areas each having a power of two size. We start from the largest + * one (i.e highest order bit set in the size). + */ +static void mvebu_pcie_add_windows(struct mvebu_pcie_port *port, + unsigned int target, unsigned int attribute, + phys_addr_t base, size_t size, + phys_addr_t remap) +{ + size_t size_mapped = 0; + + while (size) { + size_t sz = 1 << (fls(size) - 1); + int ret; + + ret = mvebu_mbus_add_window_remap_by_id(target, attribute, base, + sz, remap); + if (ret) { + dev_err(&port->pcie->pdev->dev, + "Could not create MBus window at 0x%x, size 0x%x: %d\n", + base, sz, ret); + mvebu_pcie_del_windows(port, base - size_mapped, + size_mapped); + return; + } + + size -= sz; + size_mapped += sz; + base += sz; + if (remap != MVEBU_MBUS_NO_REMAP) + remap += sz; + } +} + static void mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port) { phys_addr_t iobase; @@ -304,8 +356,8 @@ static void mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port) /* If a window was configured, remove it */ if (port->iowin_base) { - mvebu_mbus_del_window(port->iowin_base, - port->iowin_size); + mvebu_pcie_del_windows(port, port->iowin_base, + port->iowin_size); port->iowin_base = 0; port->iowin_size = 0; } @@ -333,9 +385,9 @@ static void mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port) (port->bridge.iolimitupper << 16)) - iobase) + 1; - mvebu_mbus_add_window_remap_by_id(port->io_target, port->io_attr, - port->iowin_base, port->iowin_size, - iobase); + mvebu_pcie_add_windows(port, port->io_target, port->io_attr, + port->iowin_base, port->iowin_size, + iobase); } static void mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port) @@ -346,8 +398,8 @@ static void mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port) /* If a window was configured, remove it */ if (port->memwin_base) { - mvebu_mbus_del_window(port->memwin_base, - port->memwin_size); + mvebu_pcie_del_windows(port, port->memwin_base, + port->memwin_size); port->memwin_base = 0; port->memwin_size = 0; } @@ -366,8 +418,9 @@ static void mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port) (((port->bridge.memlimit & 0xFFF0) << 16) | 0xFFFFF) - port->memwin_base + 1; - mvebu_mbus_add_window_by_id(port->mem_target, port->mem_attr, - port->memwin_base, port->memwin_size); + mvebu_pcie_add_windows(port, port->mem_target, port->mem_attr, + port->memwin_base, port->memwin_size, + MVEBU_MBUS_NO_REMAP); } /* @@ -743,14 +796,21 @@ static resource_size_t mvebu_pcie_align_resource(struct pci_dev *dev, /* * On the PCI-to-PCI bridge side, the I/O windows must have at - * least a 64 KB size and be aligned on their size, and the - * memory windows must have at least a 1 MB size and be - * aligned on their size + * least a 64 KB size and the memory windows must have at + * least a 1 MB size. Moreover, MBus windows need to have a + * base address aligned on their size, and their size must be + * a power of two. This means that if the BAR doesn't have a + * power of two size, several MBus windows will actually be + * created. We need to ensure that the biggest MBus window + * (which will be the first one) is aligned on its size, which + * explains the rounddown_pow_of_two() being done here. */ if (res->flags & IORESOURCE_IO) - return round_up(start, max_t(resource_size_t, SZ_64K, size)); + return round_up(start, max_t(resource_size_t, SZ_64K, + rounddown_pow_of_two(size))); else if (res->flags & IORESOURCE_MEM) - return round_up(start, max_t(resource_size_t, SZ_1M, size)); + return round_up(start, max_t(resource_size_t, SZ_1M, + rounddown_pow_of_two(size))); else return start; }