From c8e858fe72230dd2ad07abcbec7c9f201672a8b4 Mon Sep 17 00:00:00 2001 From: adam radford Date: Sat, 8 Oct 2011 18:15:13 -0700 Subject: [PATCH] [SCSI] megaraid_sas: Add multiple MSI-X vector/multiple reply queue support Signed-off-by: Adam Radford Signed-off-by: James Bottomley --- drivers/scsi/megaraid/megaraid_sas.h | 11 +- drivers/scsi/megaraid/megaraid_sas_base.c | 156 +++++++++++++++----- drivers/scsi/megaraid/megaraid_sas_fusion.c | 65 +++++--- drivers/scsi/megaraid/megaraid_sas_fusion.h | 3 +- 4 files changed, 176 insertions(+), 59 deletions(-) diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h index 6b3562554937..9ffb92daa58b 100644 --- a/drivers/scsi/megaraid/megaraid_sas.h +++ b/drivers/scsi/megaraid/megaraid_sas.h @@ -757,6 +757,7 @@ struct megasas_ctrl_info { #define MEGASAS_INT_CMDS 32 #define MEGASAS_SKINNY_INT_CMDS 5 +#define MEGASAS_MAX_MSIX_QUEUES 16 /* * FW can accept both 32 and 64 bit SGLs. We want to allocate 32/64 bit * SGLs based on the size of dma_addr_t @@ -1278,6 +1279,11 @@ struct megasas_aen_event { struct megasas_instance *instance; }; +struct megasas_irq_context { + struct megasas_instance *instance; + u32 MSIxIndex; +}; + struct megasas_instance { u32 *producer; @@ -1351,8 +1357,9 @@ struct megasas_instance { /* Ptr to hba specific information */ void *ctrl_context; - u8 msi_flag; - struct msix_entry msixentry; + unsigned int msix_vectors; + struct msix_entry msixentry[MEGASAS_MAX_MSIX_QUEUES]; + struct megasas_irq_context irq_context[MEGASAS_MAX_MSIX_QUEUES]; u64 map_id; struct megasas_cmd *map_update_cmd; unsigned long bar; diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index f2cf768c896d..ec09d5c2ed90 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -2536,7 +2536,7 @@ megasas_deplete_reply_queue(struct megasas_instance *instance, instance->reg_set) ) == 0) { /* Hardware may not set outbound_intr_status in MSI-X mode */ - if (!instance->msi_flag) + if (!instance->msix_vectors) return IRQ_NONE; } @@ -2594,16 +2594,14 @@ megasas_deplete_reply_queue(struct megasas_instance *instance, */ static irqreturn_t megasas_isr(int irq, void *devp) { - struct megasas_instance *instance; + struct megasas_irq_context *irq_context = devp; + struct megasas_instance *instance = irq_context->instance; unsigned long flags; irqreturn_t rc; - if (atomic_read( - &(((struct megasas_instance *)devp)->fw_reset_no_pci_access))) + if (atomic_read(&instance->fw_reset_no_pci_access)) return IRQ_HANDLED; - instance = (struct megasas_instance *)devp; - spin_lock_irqsave(&instance->hba_lock, flags); rc = megasas_deplete_reply_queue(instance, DID_OK); spin_unlock_irqrestore(&instance->hba_lock, flags); @@ -3488,6 +3486,7 @@ static int megasas_init_fw(struct megasas_instance *instance) struct megasas_register_set __iomem *reg_set; struct megasas_ctrl_info *ctrl_info; unsigned long bar_list; + int i; /* Find first memory bar */ bar_list = pci_select_bars(instance->pdev, IORESOURCE_MEM); @@ -3541,9 +3540,33 @@ static int megasas_init_fw(struct megasas_instance *instance) /* Check if MSI-X is supported while in ready state */ msix_enable = (instance->instancet->read_fw_status_reg(reg_set) & 0x4000000) >> 0x1a; - if (msix_enable && !msix_disable && - !pci_enable_msix(instance->pdev, &instance->msixentry, 1)) - instance->msi_flag = 1; + if (msix_enable && !msix_disable) { + /* Check max MSI-X vectors */ + if ((instance->pdev->device == PCI_DEVICE_ID_LSI_FUSION) || + (instance->pdev->device == PCI_DEVICE_ID_LSI_INVADER)) { + instance->msix_vectors = (readl(&instance->reg_set-> + outbound_scratch_pad_2 + ) & 0x1F) + 1; + } else + instance->msix_vectors = 1; + /* Don't bother allocating more MSI-X vectors than cpus */ + instance->msix_vectors = min(instance->msix_vectors, + (unsigned int)num_online_cpus()); + for (i = 0; i < instance->msix_vectors; i++) + instance->msixentry[i].entry = i; + i = pci_enable_msix(instance->pdev, instance->msixentry, + instance->msix_vectors); + if (i >= 0) { + if (i) { + if (!pci_enable_msix(instance->pdev, + instance->msixentry, i)) + instance->msix_vectors = i; + else + instance->msix_vectors = 0; + } + } else + instance->msix_vectors = 0; + } /* Get operational params, sge flags, send init cmd to controller */ if (instance->instancet->init_adapter(instance)) @@ -3958,7 +3981,7 @@ fail_set_dma_mask: static int __devinit megasas_probe_one(struct pci_dev *pdev, const struct pci_device_id *id) { - int rval, pos; + int rval, pos, i, j; struct Scsi_Host *host; struct megasas_instance *instance; u16 control = 0; @@ -4126,11 +4149,32 @@ megasas_probe_one(struct pci_dev *pdev, const struct pci_device_id *id) /* * Register IRQ */ - if (request_irq(instance->msi_flag ? instance->msixentry.vector : - pdev->irq, instance->instancet->service_isr, - IRQF_SHARED, "megasas", instance)) { - printk(KERN_DEBUG "megasas: Failed to register IRQ\n"); - goto fail_irq; + if (instance->msix_vectors) { + for (i = 0 ; i < instance->msix_vectors; i++) { + instance->irq_context[i].instance = instance; + instance->irq_context[i].MSIxIndex = i; + if (request_irq(instance->msixentry[i].vector, + instance->instancet->service_isr, 0, + "megasas", + &instance->irq_context[i])) { + printk(KERN_DEBUG "megasas: Failed to " + "register IRQ for vector %d.\n", i); + for (j = 0 ; j < i ; j++) + free_irq( + instance->msixentry[j].vector, + &instance->irq_context[j]); + goto fail_irq; + } + } + } else { + instance->irq_context[0].instance = instance; + instance->irq_context[0].MSIxIndex = 0; + if (request_irq(pdev->irq, instance->instancet->service_isr, + IRQF_SHARED, "megasas", + &instance->irq_context[0])) { + printk(KERN_DEBUG "megasas: Failed to register IRQ\n"); + goto fail_irq; + } } instance->instancet->enable_intr(instance->reg_set); @@ -4174,8 +4218,12 @@ megasas_probe_one(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev, NULL); instance->instancet->disable_intr(instance->reg_set); - free_irq(instance->msi_flag ? instance->msixentry.vector : - instance->pdev->irq, instance); + if (instance->msix_vectors) + for (i = 0 ; i < instance->msix_vectors; i++) + free_irq(instance->msixentry[i].vector, + &instance->irq_context[i]); + else + free_irq(instance->pdev->irq, &instance->irq_context[0]); fail_irq: if ((instance->pdev->device == PCI_DEVICE_ID_LSI_FUSION) || (instance->pdev->device == PCI_DEVICE_ID_LSI_INVADER)) @@ -4183,7 +4231,7 @@ fail_irq: else megasas_release_mfi(instance); fail_init_mfi: - if (instance->msi_flag) + if (instance->msix_vectors) pci_disable_msix(instance->pdev); fail_alloc_dma_buf: if (instance->evt_detail) @@ -4299,6 +4347,7 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state) { struct Scsi_Host *host; struct megasas_instance *instance; + int i; instance = pci_get_drvdata(pdev); host = instance->host; @@ -4322,9 +4371,14 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state) pci_set_drvdata(instance->pdev, instance); instance->instancet->disable_intr(instance->reg_set); - free_irq(instance->msi_flag ? instance->msixentry.vector : - instance->pdev->irq, instance); - if (instance->msi_flag) + + if (instance->msix_vectors) + for (i = 0 ; i < instance->msix_vectors; i++) + free_irq(instance->msixentry[i].vector, + &instance->irq_context[i]); + else + free_irq(instance->pdev->irq, &instance->irq_context[0]); + if (instance->msix_vectors) pci_disable_msix(instance->pdev); pci_save_state(pdev); @@ -4342,7 +4396,7 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state) static int megasas_resume(struct pci_dev *pdev) { - int rval; + int rval, i, j; struct Scsi_Host *host; struct megasas_instance *instance; @@ -4380,8 +4434,9 @@ megasas_resume(struct pci_dev *pdev) goto fail_ready_state; /* Now re-enable MSI-X */ - if (instance->msi_flag) - pci_enable_msix(instance->pdev, &instance->msixentry, 1); + if (instance->msix_vectors) + pci_enable_msix(instance->pdev, instance->msixentry, + instance->msix_vectors); switch (instance->pdev->device) { case PCI_DEVICE_ID_LSI_FUSION: @@ -4411,11 +4466,32 @@ megasas_resume(struct pci_dev *pdev) /* * Register IRQ */ - if (request_irq(instance->msi_flag ? instance->msixentry.vector : - pdev->irq, instance->instancet->service_isr, - IRQF_SHARED, "megasas", instance)) { - printk(KERN_ERR "megasas: Failed to register IRQ\n"); - goto fail_irq; + if (instance->msix_vectors) { + for (i = 0 ; i < instance->msix_vectors; i++) { + instance->irq_context[i].instance = instance; + instance->irq_context[i].MSIxIndex = i; + if (request_irq(instance->msixentry[i].vector, + instance->instancet->service_isr, 0, + "megasas", + &instance->irq_context[i])) { + printk(KERN_DEBUG "megasas: Failed to " + "register IRQ for vector %d.\n", i); + for (j = 0 ; j < i ; j++) + free_irq( + instance->msixentry[j].vector, + &instance->irq_context[j]); + goto fail_irq; + } + } + } else { + instance->irq_context[0].instance = instance; + instance->irq_context[0].MSIxIndex = 0; + if (request_irq(pdev->irq, instance->instancet->service_isr, + IRQF_SHARED, "megasas", + &instance->irq_context[0])) { + printk(KERN_DEBUG "megasas: Failed to register IRQ\n"); + goto fail_irq; + } } instance->instancet->enable_intr(instance->reg_set); @@ -4512,9 +4588,13 @@ static void __devexit megasas_detach_one(struct pci_dev *pdev) instance->instancet->disable_intr(instance->reg_set); - free_irq(instance->msi_flag ? instance->msixentry.vector : - instance->pdev->irq, instance); - if (instance->msi_flag) + if (instance->msix_vectors) + for (i = 0 ; i < instance->msix_vectors; i++) + free_irq(instance->msixentry[i].vector, + &instance->irq_context[i]); + else + free_irq(instance->pdev->irq, &instance->irq_context[0]); + if (instance->msix_vectors) pci_disable_msix(instance->pdev); switch (instance->pdev->device) { @@ -4560,14 +4640,20 @@ static void __devexit megasas_detach_one(struct pci_dev *pdev) */ static void megasas_shutdown(struct pci_dev *pdev) { + int i; struct megasas_instance *instance = pci_get_drvdata(pdev); + instance->unload = 1; megasas_flush_cache(instance); megasas_shutdown_controller(instance, MR_DCMD_CTRL_SHUTDOWN); instance->instancet->disable_intr(instance->reg_set); - free_irq(instance->msi_flag ? instance->msixentry.vector : - instance->pdev->irq, instance); - if (instance->msi_flag) + if (instance->msix_vectors) + for (i = 0 ; i < instance->msix_vectors; i++) + free_irq(instance->msixentry[i].vector, + &instance->irq_context[i]); + else + free_irq(instance->pdev->irq, &instance->irq_context[0]); + if (instance->msix_vectors) pci_disable_msix(instance->pdev); } diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c index ad6bd061a0f8..bfd87fab39aa 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.c +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c @@ -385,7 +385,7 @@ static int megasas_create_frame_pool_fusion(struct megasas_instance *instance) int megasas_alloc_cmds_fusion(struct megasas_instance *instance) { - int i, j; + int i, j, count; u32 max_cmd, io_frames_sz; struct fusion_context *fusion; struct megasas_cmd_fusion *cmd; @@ -409,9 +409,10 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance) goto fail_req_desc; } + count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; fusion->reply_frames_desc_pool = pci_pool_create("reply_frames pool", instance->pdev, - fusion->reply_alloc_sz, 16, 0); + fusion->reply_alloc_sz * count, 16, 0); if (!fusion->reply_frames_desc_pool) { printk(KERN_ERR "megasas; Could not allocate memory for " @@ -430,7 +431,7 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance) } reply_desc = fusion->reply_frames_desc; - for (i = 0; i < fusion->reply_q_depth; i++, reply_desc++) + for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++) reply_desc->Words = ULLONG_MAX; io_frames_sz = fusion->io_frames_alloc_sz; @@ -633,7 +634,9 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) fusion->reply_frames_desc_phys; IOCInitMessage->SystemRequestFrameBaseAddress = fusion->io_request_frames_phys; - + /* Set to 0 for none or 1 MSI-X vectors */ + IOCInitMessage->HostMSIxVectors = (instance->msix_vectors > 0 ? + instance->msix_vectors : 0); init_frame = (struct megasas_init_frame *)cmd->frame; memset(init_frame, 0, MEGAMFI_FRAME_SIZE); @@ -877,7 +880,7 @@ megasas_init_adapter_fusion(struct megasas_instance *instance) struct megasas_register_set __iomem *reg_set; struct fusion_context *fusion; u32 max_cmd; - int i = 0; + int i = 0, count; fusion = instance->ctrl_context; @@ -929,7 +932,9 @@ megasas_init_adapter_fusion(struct megasas_instance *instance) (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE - sizeof(union MPI2_SGE_IO_UNION))/16; - fusion->last_reply_idx = 0; + count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; + for (i = 0 ; i < count; i++) + fusion->last_reply_idx[i] = 0; /* * Allocate memory for descriptors @@ -1421,6 +1426,12 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, fp_possible = io_info.fpOkForIo; } + /* Use smp_processor_id() for now until cmd->request->cpu is CPU + id by default, not CPU group id, otherwise all MSI-X queues won't + be utilized */ + cmd->request_desc->SCSIIO.MSIxIndex = instance->msix_vectors ? + smp_processor_id() % instance->msix_vectors : 0; + if (fp_possible) { megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp, local_map_ptr, start_lba_lo); @@ -1691,7 +1702,7 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance, * Completes all commands that is in reply descriptor queue */ int -complete_cmd_fusion(struct megasas_instance *instance) +complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) { union MPI2_REPLY_DESCRIPTORS_UNION *desc; struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc; @@ -1711,7 +1722,9 @@ complete_cmd_fusion(struct megasas_instance *instance) return IRQ_HANDLED; desc = fusion->reply_frames_desc; - desc += fusion->last_reply_idx; + desc += ((MSIxIndex * fusion->reply_alloc_sz)/ + sizeof(union MPI2_REPLY_DESCRIPTORS_UNION)) + + fusion->last_reply_idx[MSIxIndex]; reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc; @@ -1784,16 +1797,19 @@ complete_cmd_fusion(struct megasas_instance *instance) break; } - fusion->last_reply_idx++; - if (fusion->last_reply_idx >= fusion->reply_q_depth) - fusion->last_reply_idx = 0; + fusion->last_reply_idx[MSIxIndex]++; + if (fusion->last_reply_idx[MSIxIndex] >= + fusion->reply_q_depth) + fusion->last_reply_idx[MSIxIndex] = 0; desc->Words = ULLONG_MAX; num_completed++; /* Get the next reply descriptor */ - if (!fusion->last_reply_idx) - desc = fusion->reply_frames_desc; + if (!fusion->last_reply_idx[MSIxIndex]) + desc = fusion->reply_frames_desc + + ((MSIxIndex * fusion->reply_alloc_sz)/ + sizeof(union MPI2_REPLY_DESCRIPTORS_UNION)); else desc++; @@ -1813,7 +1829,7 @@ complete_cmd_fusion(struct megasas_instance *instance) return IRQ_NONE; wmb(); - writel(fusion->last_reply_idx, + writel((MSIxIndex << 24) | fusion->last_reply_idx[MSIxIndex], &instance->reg_set->reply_post_host_index); megasas_check_and_restore_queue_depth(instance); return IRQ_HANDLED; @@ -1831,6 +1847,9 @@ megasas_complete_cmd_dpc_fusion(unsigned long instance_addr) struct megasas_instance *instance = (struct megasas_instance *)instance_addr; unsigned long flags; + u32 count, MSIxIndex; + + count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; /* If we have already declared adapter dead, donot complete cmds */ spin_lock_irqsave(&instance->hba_lock, flags); @@ -1841,7 +1860,8 @@ megasas_complete_cmd_dpc_fusion(unsigned long instance_addr) spin_unlock_irqrestore(&instance->hba_lock, flags); spin_lock_irqsave(&instance->completion_lock, flags); - complete_cmd_fusion(instance); + for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) + complete_cmd_fusion(instance, MSIxIndex); spin_unlock_irqrestore(&instance->completion_lock, flags); } @@ -1850,10 +1870,11 @@ megasas_complete_cmd_dpc_fusion(unsigned long instance_addr) */ irqreturn_t megasas_isr_fusion(int irq, void *devp) { - struct megasas_instance *instance = (struct megasas_instance *)devp; + struct megasas_irq_context *irq_context = devp; + struct megasas_instance *instance = irq_context->instance; u32 mfiStatus, fw_state; - if (!instance->msi_flag) { + if (!instance->msix_vectors) { mfiStatus = instance->instancet->clear_intr(instance->reg_set); if (!mfiStatus) return IRQ_NONE; @@ -1865,7 +1886,7 @@ irqreturn_t megasas_isr_fusion(int irq, void *devp) return IRQ_HANDLED; } - if (!complete_cmd_fusion(instance)) { + if (!complete_cmd_fusion(instance, irq_context->MSIxIndex)) { instance->instancet->clear_intr(instance->reg_set); /* If we didn't complete any commands, check for FW fault */ fw_state = instance->instancet->read_fw_status_reg( @@ -2081,14 +2102,16 @@ out: void megasas_reset_reply_desc(struct megasas_instance *instance) { - int i; + int i, count; struct fusion_context *fusion; union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc; fusion = instance->ctrl_context; - fusion->last_reply_idx = 0; + count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; + for (i = 0 ; i < count ; i++) + fusion->last_reply_idx[i] = 0; reply_desc = fusion->reply_frames_desc; - for (i = 0 ; i < fusion->reply_q_depth; i++, reply_desc++) + for (i = 0 ; i < fusion->reply_q_depth * count; i++, reply_desc++) reply_desc->Words = ULLONG_MAX; } diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.h b/drivers/scsi/megaraid/megaraid_sas_fusion.h index 9556c08490b2..088c9f91da95 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.h +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.h @@ -43,6 +43,7 @@ #define HOST_DIAG_WRITE_ENABLE 0x80 #define HOST_DIAG_RESET_ADAPTER 0x4 #define MEGASAS_FUSION_MAX_RESET_TRIES 3 +#define MAX_MSIX_QUEUES_FUSION 16 /* Invader defines */ #define MPI2_TYPE_CUDA 0x2 @@ -673,7 +674,7 @@ struct fusion_context { union MPI2_REPLY_DESCRIPTORS_UNION *reply_frames_desc; struct dma_pool *reply_frames_desc_pool; - u16 last_reply_idx; + u16 last_reply_idx[MAX_MSIX_QUEUES_FUSION]; u32 reply_q_depth; u32 request_alloc_sz;