From cc78076af14e1478c1a8fb18997674b5f8cbe3c8 Mon Sep 17 00:00:00 2001 From: Mike Marciniszyn Date: Mon, 10 Jun 2019 12:28:18 -0400 Subject: [PATCH] IB/hfi1: Correct tid qp rcd to match verbs context The qp priv rcd pointer doesn't match the context being used for verbs causing issues when 9B and kdeth packets are processed by different receive contexts and hence different CPUs. When running on different CPUs the following panic can occur: WARNING: CPU: 3 PID: 2584 at lib/list_debug.c:59 __list_del_entry+0xa1/0xd0 list_del corruption. prev->next should be ffff9a7ac31f7a30, but was ffff9a7c3bc89230 CPU: 3 PID: 2584 Comm: z_wr_iss Kdump: loaded Tainted: P OE ------------ 3.10.0-862.2.3.el7_lustre.x86_64 #1 Call Trace: [] dump_stack+0x19/0x1b [] __warn+0xd8/0x100 [] warn_slowpath_fmt+0x5f/0x80 [] __list_del_entry+0xa1/0xd0 [] process_rcv_qp_work+0xb5/0x160 [hfi1] [] handle_receive_interrupt_nodma_rtail+0x20b/0x2b0 [hfi1] [] receive_context_interrupt+0x23/0x40 [hfi1] [] __handle_irq_event_percpu+0x44/0x1c0 [] handle_irq_event_percpu+0x32/0x80 [] handle_irq_event+0x3c/0x60 [] handle_edge_irq+0x7f/0x150 [] handle_irq+0xe4/0x1a0 [] do_IRQ+0x4d/0xf0 [] common_interrupt+0x162/0x162 [] ? memcpy+0x6/0x110 [] ? abd_copy_from_buf_off_cb+0x1d/0x30 [zfs] [] ? abd_copy_to_buf_off_cb+0x30/0x30 [zfs] [] abd_iterate_func+0x97/0x120 [zfs] [] abd_copy_from_buf_off+0x39/0x60 [zfs] [] arc_write_ready+0x178/0x300 [zfs] [] ? mutex_lock+0x12/0x2f [] ? mutex_lock+0x12/0x2f [] zio_ready+0x65/0x3d0 [zfs] [] ? tsd_get_by_thread+0x2e/0x50 [spl] [] ? taskq_member+0x18/0x30 [spl] [] zio_execute+0xa2/0x100 [zfs] [] taskq_thread+0x2ac/0x4f0 [spl] [] ? wake_up_state+0x20/0x20 [] ? zio_taskq_member.isra.7.constprop.10+0x80/0x80 [zfs] [] ? taskq_thread_spawn+0x60/0x60 [spl] [] kthread+0xd1/0xe0 [] ? insert_kthread_work+0x40/0x40 [] ret_from_fork_nospec_begin+0x21/0x21 [] ? insert_kthread_work+0x40/0x40 Fix by reading the map entry in the same manner as the hardware so that the kdeth and verbs contexts match. Cc: Fixes: 5190f052a365 ("IB/hfi1: Allow the driver to initialize QP priv struct") Reviewed-by: Kaike Wan Signed-off-by: Mike Marciniszyn Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/chip.c | 13 +++++++++++++ drivers/infiniband/hw/hfi1/chip.h | 1 + drivers/infiniband/hw/hfi1/tid_rdma.c | 4 +--- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 4221a99ee7f4..d5b643a1d9fd 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -14031,6 +14031,19 @@ static void init_kdeth_qp(struct hfi1_devdata *dd) RCV_BTH_QP_KDETH_QP_SHIFT); } +/** + * hfi1_get_qp_map + * @dd: device data + * @idx: index to read + */ +u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx) +{ + u64 reg = read_csr(dd, RCV_QP_MAP_TABLE + (idx / 8) * 8); + + reg >>= (idx % 8) * 8; + return reg; +} + /** * init_qpmap_table * @dd - device data diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h index 4e6c3556ec48..b76cf81f927f 100644 --- a/drivers/infiniband/hw/hfi1/chip.h +++ b/drivers/infiniband/hw/hfi1/chip.h @@ -1445,6 +1445,7 @@ void clear_all_interrupts(struct hfi1_devdata *dd); void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr); void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr); void reset_interrupts(struct hfi1_devdata *dd); +u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx); /* * Interrupt source table. diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c index 6fb93032fbef..aa9c8d3ef87b 100644 --- a/drivers/infiniband/hw/hfi1/tid_rdma.c +++ b/drivers/infiniband/hw/hfi1/tid_rdma.c @@ -312,9 +312,7 @@ static struct hfi1_ctxtdata *qp_to_rcd(struct rvt_dev_info *rdi, if (qp->ibqp.qp_num == 0) ctxt = 0; else - ctxt = ((qp->ibqp.qp_num >> dd->qos_shift) % - (dd->n_krcv_queues - 1)) + 1; - + ctxt = hfi1_get_qp_map(dd, qp->ibqp.qp_num >> dd->qos_shift); return dd->rcd[ctxt]; }