cxgbit: add files for cxgbit.ko
cxgbit.h - This file contains data structure definitions for cxgbit.ko. cxgbit_lro.h - This file contains data structure definitions for LRO support. cxgbit_main.c - This file contains code for registering with iscsi target transport and cxgb4 driver. cxgbit_cm.c - This file contains code for connection management. cxgbit_target.c - This file contains code for processing iSCSI PDU. cxgbit_ddp.c - This file contains code for Direct Data Placement. (added check for NULL sg in cxgbit_set_one_ppod) Reported-by: Dan Carpenter <dan.carpenter@oracle.com> (add Kconfig and Makefile v2: added dependency on INET) Reported-by: Arnd Bergmann <arnd@arndb.de> Signed-off-by: Varun Prakash <varun@chelsio.com> Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
This commit is contained in:
Varun Prakash
Nicholas Bellinger
Родитель
d2faaefb8d
Коммит
9730ffcb89
|
|
@ -7,3 +7,5 @@ config ISCSI_TARGET
|
|||
help
|
||||
Say M here to enable the ConfigFS enabled Linux-iSCSI.org iSCSI
|
||||
Target Mode Stack.
|
||||
|
||||
source "drivers/target/iscsi/cxgbit/Kconfig"
|
||||
|
|
|
|||
|
|
@ -18,3 +18,4 @@ iscsi_target_mod-y += iscsi_target_parameters.o \
|
|||
iscsi_target_transport.o
|
||||
|
||||
obj-$(CONFIG_ISCSI_TARGET) += iscsi_target_mod.o
|
||||
obj-$(CONFIG_ISCSI_TARGET_CXGB4) += cxgbit/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
config ISCSI_TARGET_CXGB4
|
||||
tristate "Chelsio iSCSI target offload driver"
|
||||
depends on ISCSI_TARGET && CHELSIO_T4 && INET
|
||||
select CHELSIO_T4_UWIRE
|
||||
---help---
|
||||
To compile this driver as module, choose M here: the module
|
||||
will be called cxgbit.
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
|
||||
ccflags-y += -Idrivers/target/iscsi
|
||||
|
||||
obj-$(CONFIG_ISCSI_TARGET_CXGB4) += cxgbit.o
|
||||
|
||||
cxgbit-y := cxgbit_main.o cxgbit_cm.o cxgbit_target.o cxgbit_ddp.o
|
||||
|
|
@ -0,0 +1,353 @@
|
|||
/*
|
||||
* Copyright (c) 2016 Chelsio Communications, Inc.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*/
|
||||
|
||||
#ifndef __CXGBIT_H__
|
||||
#define __CXGBIT_H__
|
||||
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/idr.h>
|
||||
#include <linux/completion.h>
|
||||
#include <linux/netdevice.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/inet.h>
|
||||
#include <linux/wait.h>
|
||||
#include <linux/kref.h>
|
||||
#include <linux/timer.h>
|
||||
#include <linux/io.h>
|
||||
|
||||
#include <asm/byteorder.h>
|
||||
|
||||
#include <net/net_namespace.h>
|
||||
|
||||
#include <target/iscsi/iscsi_transport.h>
|
||||
#include <iscsi_target_parameters.h>
|
||||
#include <iscsi_target_login.h>
|
||||
|
||||
#include "t4_regs.h"
|
||||
#include "t4_msg.h"
|
||||
#include "cxgb4.h"
|
||||
#include "cxgb4_uld.h"
|
||||
#include "l2t.h"
|
||||
#include "cxgb4_ppm.h"
|
||||
#include "cxgbit_lro.h"
|
||||
|
||||
extern struct mutex cdev_list_lock;
|
||||
extern struct list_head cdev_list_head;
|
||||
struct cxgbit_np;
|
||||
|
||||
struct cxgbit_sock;
|
||||
|
||||
struct cxgbit_cmd {
|
||||
struct scatterlist sg;
|
||||
struct cxgbi_task_tag_info ttinfo;
|
||||
bool setup_ddp;
|
||||
bool release;
|
||||
};
|
||||
|
||||
#define CXGBIT_MAX_ISO_PAYLOAD \
|
||||
min_t(u32, MAX_SKB_FRAGS * PAGE_SIZE, 65535)
|
||||
|
||||
struct cxgbit_iso_info {
|
||||
u8 flags;
|
||||
u32 mpdu;
|
||||
u32 len;
|
||||
u32 burst_len;
|
||||
};
|
||||
|
||||
enum cxgbit_skcb_flags {
|
||||
SKCBF_TX_NEED_HDR = (1 << 0), /* packet needs a header */
|
||||
SKCBF_TX_FLAG_COMPL = (1 << 1), /* wr completion flag */
|
||||
SKCBF_TX_ISO = (1 << 2), /* iso cpl in tx skb */
|
||||
SKCBF_RX_LRO = (1 << 3), /* lro skb */
|
||||
};
|
||||
|
||||
struct cxgbit_skb_rx_cb {
|
||||
u8 opcode;
|
||||
void *pdu_cb;
|
||||
void (*backlog_fn)(struct cxgbit_sock *, struct sk_buff *);
|
||||
};
|
||||
|
||||
struct cxgbit_skb_tx_cb {
|
||||
u8 submode;
|
||||
u32 extra_len;
|
||||
};
|
||||
|
||||
union cxgbit_skb_cb {
|
||||
struct {
|
||||
u8 flags;
|
||||
union {
|
||||
struct cxgbit_skb_tx_cb tx;
|
||||
struct cxgbit_skb_rx_cb rx;
|
||||
};
|
||||
};
|
||||
|
||||
struct {
|
||||
/* This member must be first. */
|
||||
struct l2t_skb_cb l2t;
|
||||
struct sk_buff *wr_next;
|
||||
};
|
||||
};
|
||||
|
||||
#define CXGBIT_SKB_CB(skb) ((union cxgbit_skb_cb *)&((skb)->cb[0]))
|
||||
#define cxgbit_skcb_flags(skb) (CXGBIT_SKB_CB(skb)->flags)
|
||||
#define cxgbit_skcb_submode(skb) (CXGBIT_SKB_CB(skb)->tx.submode)
|
||||
#define cxgbit_skcb_tx_wr_next(skb) (CXGBIT_SKB_CB(skb)->wr_next)
|
||||
#define cxgbit_skcb_tx_extralen(skb) (CXGBIT_SKB_CB(skb)->tx.extra_len)
|
||||
#define cxgbit_skcb_rx_opcode(skb) (CXGBIT_SKB_CB(skb)->rx.opcode)
|
||||
#define cxgbit_skcb_rx_backlog_fn(skb) (CXGBIT_SKB_CB(skb)->rx.backlog_fn)
|
||||
#define cxgbit_rx_pdu_cb(skb) (CXGBIT_SKB_CB(skb)->rx.pdu_cb)
|
||||
|
||||
static inline void *cplhdr(struct sk_buff *skb)
|
||||
{
|
||||
return skb->data;
|
||||
}
|
||||
|
||||
enum cxgbit_cdev_flags {
|
||||
CDEV_STATE_UP = 0,
|
||||
CDEV_ISO_ENABLE,
|
||||
CDEV_DDP_ENABLE,
|
||||
};
|
||||
|
||||
#define NP_INFO_HASH_SIZE 32
|
||||
|
||||
struct np_info {
|
||||
struct np_info *next;
|
||||
struct cxgbit_np *cnp;
|
||||
unsigned int stid;
|
||||
};
|
||||
|
||||
struct cxgbit_list_head {
|
||||
struct list_head list;
|
||||
/* device lock */
|
||||
spinlock_t lock;
|
||||
};
|
||||
|
||||
struct cxgbit_device {
|
||||
struct list_head list;
|
||||
struct cxgb4_lld_info lldi;
|
||||
struct np_info *np_hash_tab[NP_INFO_HASH_SIZE];
|
||||
/* np lock */
|
||||
spinlock_t np_lock;
|
||||
u8 selectq[MAX_NPORTS][2];
|
||||
struct cxgbit_list_head cskq;
|
||||
u32 mdsl;
|
||||
struct kref kref;
|
||||
unsigned long flags;
|
||||
};
|
||||
|
||||
struct cxgbit_wr_wait {
|
||||
struct completion completion;
|
||||
int ret;
|
||||
};
|
||||
|
||||
enum cxgbit_csk_state {
|
||||
CSK_STATE_IDLE = 0,
|
||||
CSK_STATE_LISTEN,
|
||||
CSK_STATE_CONNECTING,
|
||||
CSK_STATE_ESTABLISHED,
|
||||
CSK_STATE_ABORTING,
|
||||
CSK_STATE_CLOSING,
|
||||
CSK_STATE_MORIBUND,
|
||||
CSK_STATE_DEAD,
|
||||
};
|
||||
|
||||
enum cxgbit_csk_flags {
|
||||
CSK_TX_DATA_SENT = 0,
|
||||
CSK_LOGIN_PDU_DONE,
|
||||
CSK_LOGIN_DONE,
|
||||
CSK_DDP_ENABLE,
|
||||
};
|
||||
|
||||
struct cxgbit_sock_common {
|
||||
struct cxgbit_device *cdev;
|
||||
struct sockaddr_storage local_addr;
|
||||
struct sockaddr_storage remote_addr;
|
||||
struct cxgbit_wr_wait wr_wait;
|
||||
enum cxgbit_csk_state state;
|
||||
unsigned long flags;
|
||||
};
|
||||
|
||||
struct cxgbit_np {
|
||||
struct cxgbit_sock_common com;
|
||||
wait_queue_head_t accept_wait;
|
||||
struct iscsi_np *np;
|
||||
struct completion accept_comp;
|
||||
struct list_head np_accept_list;
|
||||
/* np accept lock */
|
||||
spinlock_t np_accept_lock;
|
||||
struct kref kref;
|
||||
unsigned int stid;
|
||||
};
|
||||
|
||||
struct cxgbit_sock {
|
||||
struct cxgbit_sock_common com;
|
||||
struct cxgbit_np *cnp;
|
||||
struct iscsi_conn *conn;
|
||||
struct l2t_entry *l2t;
|
||||
struct dst_entry *dst;
|
||||
struct list_head list;
|
||||
struct sk_buff_head rxq;
|
||||
struct sk_buff_head txq;
|
||||
struct sk_buff_head ppodq;
|
||||
struct sk_buff_head backlogq;
|
||||
struct sk_buff_head skbq;
|
||||
struct sk_buff *wr_pending_head;
|
||||
struct sk_buff *wr_pending_tail;
|
||||
struct sk_buff *skb;
|
||||
struct sk_buff *lro_skb;
|
||||
struct sk_buff *lro_hskb;
|
||||
struct list_head accept_node;
|
||||
/* socket lock */
|
||||
spinlock_t lock;
|
||||
wait_queue_head_t waitq;
|
||||
wait_queue_head_t ack_waitq;
|
||||
bool lock_owner;
|
||||
struct kref kref;
|
||||
u32 max_iso_npdu;
|
||||
u32 wr_cred;
|
||||
u32 wr_una_cred;
|
||||
u32 wr_max_cred;
|
||||
u32 snd_una;
|
||||
u32 tid;
|
||||
u32 snd_nxt;
|
||||
u32 rcv_nxt;
|
||||
u32 smac_idx;
|
||||
u32 tx_chan;
|
||||
u32 mtu;
|
||||
u32 write_seq;
|
||||
u32 rx_credits;
|
||||
u32 snd_win;
|
||||
u32 rcv_win;
|
||||
u16 mss;
|
||||
u16 emss;
|
||||
u16 plen;
|
||||
u16 rss_qid;
|
||||
u16 txq_idx;
|
||||
u16 ctrlq_idx;
|
||||
u8 tos;
|
||||
u8 port_id;
|
||||
#define CXGBIT_SUBMODE_HCRC 0x1
|
||||
#define CXGBIT_SUBMODE_DCRC 0x2
|
||||
u8 submode;
|
||||
#ifdef CONFIG_CHELSIO_T4_DCB
|
||||
u8 dcb_priority;
|
||||
#endif
|
||||
u8 snd_wscale;
|
||||
};
|
||||
|
||||
void _cxgbit_free_cdev(struct kref *kref);
|
||||
void _cxgbit_free_csk(struct kref *kref);
|
||||
void _cxgbit_free_cnp(struct kref *kref);
|
||||
|
||||
static inline void cxgbit_get_cdev(struct cxgbit_device *cdev)
|
||||
{
|
||||
kref_get(&cdev->kref);
|
||||
}
|
||||
|
||||
static inline void cxgbit_put_cdev(struct cxgbit_device *cdev)
|
||||
{
|
||||
kref_put(&cdev->kref, _cxgbit_free_cdev);
|
||||
}
|
||||
|
||||
static inline void cxgbit_get_csk(struct cxgbit_sock *csk)
|
||||
{
|
||||
kref_get(&csk->kref);
|
||||
}
|
||||
|
||||
static inline void cxgbit_put_csk(struct cxgbit_sock *csk)
|
||||
{
|
||||
kref_put(&csk->kref, _cxgbit_free_csk);
|
||||
}
|
||||
|
||||
static inline void cxgbit_get_cnp(struct cxgbit_np *cnp)
|
||||
{
|
||||
kref_get(&cnp->kref);
|
||||
}
|
||||
|
||||
static inline void cxgbit_put_cnp(struct cxgbit_np *cnp)
|
||||
{
|
||||
kref_put(&cnp->kref, _cxgbit_free_cnp);
|
||||
}
|
||||
|
||||
static inline void cxgbit_sock_reset_wr_list(struct cxgbit_sock *csk)
|
||||
{
|
||||
csk->wr_pending_tail = NULL;
|
||||
csk->wr_pending_head = NULL;
|
||||
}
|
||||
|
||||
static inline struct sk_buff *cxgbit_sock_peek_wr(const struct cxgbit_sock *csk)
|
||||
{
|
||||
return csk->wr_pending_head;
|
||||
}
|
||||
|
||||
static inline void
|
||||
cxgbit_sock_enqueue_wr(struct cxgbit_sock *csk, struct sk_buff *skb)
|
||||
{
|
||||
cxgbit_skcb_tx_wr_next(skb) = NULL;
|
||||
|
||||
skb_get(skb);
|
||||
|
||||
if (!csk->wr_pending_head)
|
||||
csk->wr_pending_head = skb;
|
||||
else
|
||||
cxgbit_skcb_tx_wr_next(csk->wr_pending_tail) = skb;
|
||||
csk->wr_pending_tail = skb;
|
||||
}
|
||||
|
||||
static inline struct sk_buff *cxgbit_sock_dequeue_wr(struct cxgbit_sock *csk)
|
||||
{
|
||||
struct sk_buff *skb = csk->wr_pending_head;
|
||||
|
||||
if (likely(skb)) {
|
||||
csk->wr_pending_head = cxgbit_skcb_tx_wr_next(skb);
|
||||
cxgbit_skcb_tx_wr_next(skb) = NULL;
|
||||
}
|
||||
return skb;
|
||||
}
|
||||
|
||||
typedef void (*cxgbit_cplhandler_func)(struct cxgbit_device *,
|
||||
struct sk_buff *);
|
||||
|
||||
int cxgbit_setup_np(struct iscsi_np *, struct sockaddr_storage *);
|
||||
int cxgbit_setup_conn_digest(struct cxgbit_sock *);
|
||||
int cxgbit_accept_np(struct iscsi_np *, struct iscsi_conn *);
|
||||
void cxgbit_free_np(struct iscsi_np *);
|
||||
void cxgbit_free_conn(struct iscsi_conn *);
|
||||
extern cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS];
|
||||
int cxgbit_get_login_rx(struct iscsi_conn *, struct iscsi_login *);
|
||||
int cxgbit_rx_data_ack(struct cxgbit_sock *);
|
||||
int cxgbit_l2t_send(struct cxgbit_device *, struct sk_buff *,
|
||||
struct l2t_entry *);
|
||||
void cxgbit_push_tx_frames(struct cxgbit_sock *);
|
||||
int cxgbit_put_login_tx(struct iscsi_conn *, struct iscsi_login *, u32);
|
||||
int cxgbit_xmit_pdu(struct iscsi_conn *, struct iscsi_cmd *,
|
||||
struct iscsi_datain_req *, const void *, u32);
|
||||
void cxgbit_get_r2t_ttt(struct iscsi_conn *, struct iscsi_cmd *,
|
||||
struct iscsi_r2t *);
|
||||
u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *);
|
||||
int cxgbit_ofld_send(struct cxgbit_device *, struct sk_buff *);
|
||||
void cxgbit_get_rx_pdu(struct iscsi_conn *);
|
||||
int cxgbit_validate_params(struct iscsi_conn *);
|
||||
struct cxgbit_device *cxgbit_find_device(struct net_device *, u8 *);
|
||||
|
||||
/* DDP */
|
||||
int cxgbit_ddp_init(struct cxgbit_device *);
|
||||
int cxgbit_setup_conn_pgidx(struct cxgbit_sock *, u32);
|
||||
int cxgbit_reserve_ttt(struct cxgbit_sock *, struct iscsi_cmd *);
|
||||
void cxgbit_release_cmd(struct iscsi_conn *, struct iscsi_cmd *);
|
||||
|
||||
static inline
|
||||
struct cxgbi_ppm *cdev2ppm(struct cxgbit_device *cdev)
|
||||
{
|
||||
return (struct cxgbi_ppm *)(*cdev->lldi.iscsi_ppm);
|
||||
}
|
||||
#endif /* __CXGBIT_H__ */
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,325 @@
|
|||
/*
|
||||
* Copyright (c) 2016 Chelsio Communications, Inc.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*/
|
||||
|
||||
#include "cxgbit.h"
|
||||
|
||||
static void
|
||||
cxgbit_set_one_ppod(struct cxgbi_pagepod *ppod,
|
||||
struct cxgbi_task_tag_info *ttinfo,
|
||||
struct scatterlist **sg_pp, unsigned int *sg_off)
|
||||
{
|
||||
struct scatterlist *sg = sg_pp ? *sg_pp : NULL;
|
||||
unsigned int offset = sg_off ? *sg_off : 0;
|
||||
dma_addr_t addr = 0UL;
|
||||
unsigned int len = 0;
|
||||
int i;
|
||||
|
||||
memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr));
|
||||
|
||||
if (sg) {
|
||||
addr = sg_dma_address(sg);
|
||||
len = sg_dma_len(sg);
|
||||
}
|
||||
|
||||
for (i = 0; i < PPOD_PAGES_MAX; i++) {
|
||||
if (sg) {
|
||||
ppod->addr[i] = cpu_to_be64(addr + offset);
|
||||
offset += PAGE_SIZE;
|
||||
if (offset == (len + sg->offset)) {
|
||||
offset = 0;
|
||||
sg = sg_next(sg);
|
||||
if (sg) {
|
||||
addr = sg_dma_address(sg);
|
||||
len = sg_dma_len(sg);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ppod->addr[i] = 0ULL;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* the fifth address needs to be repeated in the next ppod, so do
|
||||
* not move sg
|
||||
*/
|
||||
if (sg_pp) {
|
||||
*sg_pp = sg;
|
||||
*sg_off = offset;
|
||||
}
|
||||
|
||||
if (offset == len) {
|
||||
offset = 0;
|
||||
if (sg) {
|
||||
sg = sg_next(sg);
|
||||
if (sg)
|
||||
addr = sg_dma_address(sg);
|
||||
}
|
||||
}
|
||||
ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL;
|
||||
}
|
||||
|
||||
static struct sk_buff *
|
||||
cxgbit_ppod_init_idata(struct cxgbit_device *cdev, struct cxgbi_ppm *ppm,
|
||||
unsigned int idx, unsigned int npods, unsigned int tid)
|
||||
{
|
||||
struct ulp_mem_io *req;
|
||||
struct ulptx_idata *idata;
|
||||
unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
|
||||
unsigned int dlen = npods << PPOD_SIZE_SHIFT;
|
||||
unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
|
||||
sizeof(struct ulptx_idata) + dlen, 16);
|
||||
struct sk_buff *skb;
|
||||
|
||||
skb = alloc_skb(wr_len, GFP_KERNEL);
|
||||
if (!skb)
|
||||
return NULL;
|
||||
|
||||
req = (struct ulp_mem_io *)__skb_put(skb, wr_len);
|
||||
INIT_ULPTX_WR(req, wr_len, 0, tid);
|
||||
req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) |
|
||||
FW_WR_ATOMIC_V(0));
|
||||
req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
|
||||
ULP_MEMIO_ORDER_V(0) |
|
||||
T5_ULP_MEMIO_IMM_V(1));
|
||||
req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5));
|
||||
req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5));
|
||||
req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
|
||||
|
||||
idata = (struct ulptx_idata *)(req + 1);
|
||||
idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
|
||||
idata->len = htonl(dlen);
|
||||
|
||||
return skb;
|
||||
}
|
||||
|
||||
static int
|
||||
cxgbit_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
|
||||
struct cxgbi_task_tag_info *ttinfo, unsigned int idx,
|
||||
unsigned int npods, struct scatterlist **sg_pp,
|
||||
unsigned int *sg_off)
|
||||
{
|
||||
struct cxgbit_device *cdev = csk->com.cdev;
|
||||
struct sk_buff *skb;
|
||||
struct ulp_mem_io *req;
|
||||
struct ulptx_idata *idata;
|
||||
struct cxgbi_pagepod *ppod;
|
||||
unsigned int i;
|
||||
|
||||
skb = cxgbit_ppod_init_idata(cdev, ppm, idx, npods, csk->tid);
|
||||
if (!skb)
|
||||
return -ENOMEM;
|
||||
|
||||
req = (struct ulp_mem_io *)skb->data;
|
||||
idata = (struct ulptx_idata *)(req + 1);
|
||||
ppod = (struct cxgbi_pagepod *)(idata + 1);
|
||||
|
||||
for (i = 0; i < npods; i++, ppod++)
|
||||
cxgbit_set_one_ppod(ppod, ttinfo, sg_pp, sg_off);
|
||||
|
||||
__skb_queue_tail(&csk->ppodq, skb);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
cxgbit_ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
|
||||
struct cxgbi_task_tag_info *ttinfo)
|
||||
{
|
||||
unsigned int pidx = ttinfo->idx;
|
||||
unsigned int npods = ttinfo->npods;
|
||||
unsigned int i, cnt;
|
||||
struct scatterlist *sg = ttinfo->sgl;
|
||||
unsigned int offset = 0;
|
||||
int ret = 0;
|
||||
|
||||
for (i = 0; i < npods; i += cnt, pidx += cnt) {
|
||||
cnt = npods - i;
|
||||
|
||||
if (cnt > ULPMEM_IDATA_MAX_NPPODS)
|
||||
cnt = ULPMEM_IDATA_MAX_NPPODS;
|
||||
|
||||
ret = cxgbit_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt,
|
||||
&sg, &offset);
|
||||
if (ret < 0)
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int cxgbit_ddp_sgl_check(struct scatterlist *sg,
|
||||
unsigned int nents)
|
||||
{
|
||||
unsigned int last_sgidx = nents - 1;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < nents; i++, sg = sg_next(sg)) {
|
||||
unsigned int len = sg->length + sg->offset;
|
||||
|
||||
if ((sg->offset & 0x3) || (i && sg->offset) ||
|
||||
((i != last_sgidx) && (len != PAGE_SIZE))) {
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
cxgbit_ddp_reserve(struct cxgbit_sock *csk, struct cxgbi_task_tag_info *ttinfo,
|
||||
unsigned int xferlen)
|
||||
{
|
||||
struct cxgbit_device *cdev = csk->com.cdev;
|
||||
struct cxgbi_ppm *ppm = cdev2ppm(cdev);
|
||||
struct scatterlist *sgl = ttinfo->sgl;
|
||||
unsigned int sgcnt = ttinfo->nents;
|
||||
unsigned int sg_offset = sgl->offset;
|
||||
int ret;
|
||||
|
||||
if ((xferlen < DDP_THRESHOLD) || (!sgcnt)) {
|
||||
pr_debug("ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
|
||||
ppm, ppm->tformat.pgsz_idx_dflt,
|
||||
xferlen, ttinfo->nents);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (cxgbit_ddp_sgl_check(sgl, sgcnt) < 0)
|
||||
return -EINVAL;
|
||||
|
||||
ttinfo->nr_pages = (xferlen + sgl->offset +
|
||||
(1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT;
|
||||
|
||||
/*
|
||||
* the ddp tag will be used for the ttt in the outgoing r2t pdu
|
||||
*/
|
||||
ret = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx,
|
||||
&ttinfo->tag, 0);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
ttinfo->npods = ret;
|
||||
|
||||
sgl->offset = 0;
|
||||
ret = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
|
||||
sgl->offset = sg_offset;
|
||||
if (!ret) {
|
||||
pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
|
||||
__func__, 0, xferlen, sgcnt);
|
||||
goto rel_ppods;
|
||||
}
|
||||
|
||||
cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset,
|
||||
xferlen, &ttinfo->hdr);
|
||||
|
||||
ret = cxgbit_ddp_set_map(ppm, csk, ttinfo);
|
||||
if (ret < 0) {
|
||||
__skb_queue_purge(&csk->ppodq);
|
||||
dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
|
||||
goto rel_ppods;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
rel_ppods:
|
||||
cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
void
|
||||
cxgbit_get_r2t_ttt(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
|
||||
struct iscsi_r2t *r2t)
|
||||
{
|
||||
struct cxgbit_sock *csk = conn->context;
|
||||
struct cxgbit_device *cdev = csk->com.cdev;
|
||||
struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
|
||||
struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
|
||||
int ret = -EINVAL;
|
||||
|
||||
if ((!ccmd->setup_ddp) ||
|
||||
(!test_bit(CSK_DDP_ENABLE, &csk->com.flags)))
|
||||
goto out;
|
||||
|
||||
ccmd->setup_ddp = false;
|
||||
|
||||
ttinfo->sgl = cmd->se_cmd.t_data_sg;
|
||||
ttinfo->nents = cmd->se_cmd.t_data_nents;
|
||||
|
||||
ret = cxgbit_ddp_reserve(csk, ttinfo, cmd->se_cmd.data_length);
|
||||
if (ret < 0) {
|
||||
pr_info("csk 0x%p, cmd 0x%p, xfer len %u, sgcnt %u no ddp.\n",
|
||||
csk, cmd, cmd->se_cmd.data_length, ttinfo->nents);
|
||||
|
||||
ttinfo->sgl = NULL;
|
||||
ttinfo->nents = 0;
|
||||
} else {
|
||||
ccmd->release = true;
|
||||
}
|
||||
out:
|
||||
pr_debug("cdev 0x%p, cmd 0x%p, tag 0x%x\n", cdev, cmd, ttinfo->tag);
|
||||
r2t->targ_xfer_tag = ttinfo->tag;
|
||||
}
|
||||
|
||||
void cxgbit_release_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
|
||||
{
|
||||
struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
|
||||
|
||||
if (ccmd->release) {
|
||||
struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
|
||||
|
||||
if (ttinfo->sgl) {
|
||||
struct cxgbit_sock *csk = conn->context;
|
||||
struct cxgbit_device *cdev = csk->com.cdev;
|
||||
struct cxgbi_ppm *ppm = cdev2ppm(cdev);
|
||||
|
||||
cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
|
||||
|
||||
dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl,
|
||||
ttinfo->nents, DMA_FROM_DEVICE);
|
||||
} else {
|
||||
put_page(sg_page(&ccmd->sg));
|
||||
}
|
||||
|
||||
ccmd->release = false;
|
||||
}
|
||||
}
|
||||
|
||||
int cxgbit_ddp_init(struct cxgbit_device *cdev)
|
||||
{
|
||||
struct cxgb4_lld_info *lldi = &cdev->lldi;
|
||||
struct net_device *ndev = cdev->lldi.ports[0];
|
||||
struct cxgbi_tag_format tformat;
|
||||
unsigned int ppmax;
|
||||
int ret, i;
|
||||
|
||||
if (!lldi->vr->iscsi.size) {
|
||||
pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name);
|
||||
return -EACCES;
|
||||
}
|
||||
|
||||
ppmax = lldi->vr->iscsi.size >> PPOD_SIZE_SHIFT;
|
||||
|
||||
memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
|
||||
for (i = 0; i < 4; i++)
|
||||
tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3))
|
||||
& 0xF;
|
||||
cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat);
|
||||
|
||||
ret = cxgbi_ppm_init(lldi->iscsi_ppm, cdev->lldi.ports[0],
|
||||
cdev->lldi.pdev, &cdev->lldi, &tformat,
|
||||
ppmax, lldi->iscsi_llimit,
|
||||
lldi->vr->iscsi.start, 2);
|
||||
if (ret >= 0) {
|
||||
struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*lldi->iscsi_ppm);
|
||||
|
||||
if ((ppm->tformat.pgsz_idx_dflt < DDP_PGIDX_MAX) &&
|
||||
(ppm->ppmax >= 1024))
|
||||
set_bit(CDEV_DDP_ENABLE, &cdev->flags);
|
||||
ret = 0;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Copyright (c) 2016 Chelsio Communications, Inc.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef __CXGBIT_LRO_H__
|
||||
#define __CXGBIT_LRO_H__
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/skbuff.h>
|
||||
|
||||
#define LRO_FLUSH_LEN_MAX 65535
|
||||
|
||||
struct cxgbit_lro_cb {
|
||||
struct cxgbit_sock *csk;
|
||||
u32 pdu_totallen;
|
||||
u32 offset;
|
||||
u8 pdu_idx;
|
||||
bool complete;
|
||||
};
|
||||
|
||||
enum cxgbit_pducb_flags {
|
||||
PDUCBF_RX_HDR = (1 << 0), /* received pdu header */
|
||||
PDUCBF_RX_DATA = (1 << 1), /* received pdu payload */
|
||||
PDUCBF_RX_STATUS = (1 << 2), /* received ddp status */
|
||||
PDUCBF_RX_DATA_DDPD = (1 << 3), /* pdu payload ddp'd */
|
||||
PDUCBF_RX_HCRC_ERR = (1 << 4), /* header digest error */
|
||||
PDUCBF_RX_DCRC_ERR = (1 << 5), /* data digest error */
|
||||
};
|
||||
|
||||
struct cxgbit_lro_pdu_cb {
|
||||
u8 flags;
|
||||
u8 frags;
|
||||
u8 hfrag_idx;
|
||||
u8 nr_dfrags;
|
||||
u8 dfrag_idx;
|
||||
bool complete;
|
||||
u32 seq;
|
||||
u32 pdulen;
|
||||
u32 hlen;
|
||||
u32 dlen;
|
||||
u32 doffset;
|
||||
u32 ddigest;
|
||||
void *hdr;
|
||||
};
|
||||
|
||||
#define LRO_SKB_MAX_HEADROOM \
|
||||
(sizeof(struct cxgbit_lro_cb) + \
|
||||
(MAX_SKB_FRAGS * sizeof(struct cxgbit_lro_pdu_cb)))
|
||||
|
||||
#define LRO_SKB_MIN_HEADROOM \
|
||||
(sizeof(struct cxgbit_lro_cb) + \
|
||||
sizeof(struct cxgbit_lro_pdu_cb))
|
||||
|
||||
#define cxgbit_skb_lro_cb(skb) ((struct cxgbit_lro_cb *)skb->data)
|
||||
#define cxgbit_skb_lro_pdu_cb(skb, i) \
|
||||
((struct cxgbit_lro_pdu_cb *)(skb->data + sizeof(struct cxgbit_lro_cb) \
|
||||
+ (i * sizeof(struct cxgbit_lro_pdu_cb))))
|
||||
|
||||
#define CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT 16 /* ddp'able */
|
||||
#define CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT 19 /* pad error */
|
||||
#define CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT 20 /* hcrc error */
|
||||
#define CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT 21 /* dcrc error */
|
||||
|
||||
#endif /*__CXGBIT_LRO_H_*/
|
||||
|
|
@ -0,0 +1,701 @@
|
|||
/*
|
||||
* Copyright (c) 2016 Chelsio Communications, Inc.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*/
|
||||
|
||||
#define DRV_NAME "cxgbit"
|
||||
#define DRV_VERSION "1.0.0-ko"
|
||||
#define pr_fmt(fmt) DRV_NAME ": " fmt
|
||||
|
||||
#include "cxgbit.h"
|
||||
|
||||
#ifdef CONFIG_CHELSIO_T4_DCB
|
||||
#include <net/dcbevent.h>
|
||||
#include "cxgb4_dcb.h"
|
||||
#endif
|
||||
|
||||
LIST_HEAD(cdev_list_head);
|
||||
/* cdev list lock */
|
||||
DEFINE_MUTEX(cdev_list_lock);
|
||||
|
||||
void _cxgbit_free_cdev(struct kref *kref)
|
||||
{
|
||||
struct cxgbit_device *cdev;
|
||||
|
||||
cdev = container_of(kref, struct cxgbit_device, kref);
|
||||
kfree(cdev);
|
||||
}
|
||||
|
||||
static void cxgbit_set_mdsl(struct cxgbit_device *cdev)
|
||||
{
|
||||
struct cxgb4_lld_info *lldi = &cdev->lldi;
|
||||
u32 mdsl;
|
||||
|
||||
#define ULP2_MAX_PKT_LEN 16224
|
||||
#define ISCSI_PDU_NONPAYLOAD_LEN 312
|
||||
mdsl = min_t(u32, lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN,
|
||||
ULP2_MAX_PKT_LEN - ISCSI_PDU_NONPAYLOAD_LEN);
|
||||
mdsl = min_t(u32, mdsl, 8192);
|
||||
mdsl = min_t(u32, mdsl, (MAX_SKB_FRAGS - 1) * PAGE_SIZE);
|
||||
|
||||
cdev->mdsl = mdsl;
|
||||
}
|
||||
|
||||
static void *cxgbit_uld_add(const struct cxgb4_lld_info *lldi)
|
||||
{
|
||||
struct cxgbit_device *cdev;
|
||||
|
||||
if (is_t4(lldi->adapter_type))
|
||||
return ERR_PTR(-ENODEV);
|
||||
|
||||
cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
|
||||
if (!cdev)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
kref_init(&cdev->kref);
|
||||
|
||||
cdev->lldi = *lldi;
|
||||
|
||||
cxgbit_set_mdsl(cdev);
|
||||
|
||||
if (cxgbit_ddp_init(cdev) < 0) {
|
||||
kfree(cdev);
|
||||
return ERR_PTR(-EINVAL);
|
||||
}
|
||||
|
||||
if (!test_bit(CDEV_DDP_ENABLE, &cdev->flags))
|
||||
pr_info("cdev %s ddp init failed\n",
|
||||
pci_name(lldi->pdev));
|
||||
|
||||
if (lldi->fw_vers >= 0x10d2b00)
|
||||
set_bit(CDEV_ISO_ENABLE, &cdev->flags);
|
||||
|
||||
spin_lock_init(&cdev->cskq.lock);
|
||||
INIT_LIST_HEAD(&cdev->cskq.list);
|
||||
|
||||
mutex_lock(&cdev_list_lock);
|
||||
list_add_tail(&cdev->list, &cdev_list_head);
|
||||
mutex_unlock(&cdev_list_lock);
|
||||
|
||||
pr_info("cdev %s added for iSCSI target transport\n",
|
||||
pci_name(lldi->pdev));
|
||||
|
||||
return cdev;
|
||||
}
|
||||
|
||||
static void cxgbit_close_conn(struct cxgbit_device *cdev)
|
||||
{
|
||||
struct cxgbit_sock *csk;
|
||||
struct sk_buff *skb;
|
||||
bool wakeup_thread = false;
|
||||
|
||||
spin_lock_bh(&cdev->cskq.lock);
|
||||
list_for_each_entry(csk, &cdev->cskq.list, list) {
|
||||
skb = alloc_skb(0, GFP_ATOMIC);
|
||||
if (!skb)
|
||||
continue;
|
||||
|
||||
spin_lock_bh(&csk->rxq.lock);
|
||||
__skb_queue_tail(&csk->rxq, skb);
|
||||
if (skb_queue_len(&csk->rxq) == 1)
|
||||
wakeup_thread = true;
|
||||
spin_unlock_bh(&csk->rxq.lock);
|
||||
|
||||
if (wakeup_thread) {
|
||||
wake_up(&csk->waitq);
|
||||
wakeup_thread = false;
|
||||
}
|
||||
}
|
||||
spin_unlock_bh(&cdev->cskq.lock);
|
||||
}
|
||||
|
||||
static void cxgbit_detach_cdev(struct cxgbit_device *cdev)
|
||||
{
|
||||
bool free_cdev = false;
|
||||
|
||||
spin_lock_bh(&cdev->cskq.lock);
|
||||
if (list_empty(&cdev->cskq.list))
|
||||
free_cdev = true;
|
||||
spin_unlock_bh(&cdev->cskq.lock);
|
||||
|
||||
if (free_cdev) {
|
||||
mutex_lock(&cdev_list_lock);
|
||||
list_del(&cdev->list);
|
||||
mutex_unlock(&cdev_list_lock);
|
||||
|
||||
cxgbit_put_cdev(cdev);
|
||||
} else {
|
||||
cxgbit_close_conn(cdev);
|
||||
}
|
||||
}
|
||||
|
||||
static int cxgbit_uld_state_change(void *handle, enum cxgb4_state state)
|
||||
{
|
||||
struct cxgbit_device *cdev = handle;
|
||||
|
||||
switch (state) {
|
||||
case CXGB4_STATE_UP:
|
||||
set_bit(CDEV_STATE_UP, &cdev->flags);
|
||||
pr_info("cdev %s state UP.\n", pci_name(cdev->lldi.pdev));
|
||||
break;
|
||||
case CXGB4_STATE_START_RECOVERY:
|
||||
clear_bit(CDEV_STATE_UP, &cdev->flags);
|
||||
cxgbit_close_conn(cdev);
|
||||
pr_info("cdev %s state RECOVERY.\n", pci_name(cdev->lldi.pdev));
|
||||
break;
|
||||
case CXGB4_STATE_DOWN:
|
||||
pr_info("cdev %s state DOWN.\n", pci_name(cdev->lldi.pdev));
|
||||
break;
|
||||
case CXGB4_STATE_DETACH:
|
||||
clear_bit(CDEV_STATE_UP, &cdev->flags);
|
||||
pr_info("cdev %s state DETACH.\n", pci_name(cdev->lldi.pdev));
|
||||
cxgbit_detach_cdev(cdev);
|
||||
break;
|
||||
default:
|
||||
pr_info("cdev %s unknown state %d.\n",
|
||||
pci_name(cdev->lldi.pdev), state);
|
||||
break;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
cxgbit_proc_ddp_status(unsigned int tid, struct cpl_rx_data_ddp *cpl,
|
||||
struct cxgbit_lro_pdu_cb *pdu_cb)
|
||||
{
|
||||
unsigned int status = ntohl(cpl->ddpvld);
|
||||
|
||||
pdu_cb->flags |= PDUCBF_RX_STATUS;
|
||||
pdu_cb->ddigest = ntohl(cpl->ulp_crc);
|
||||
pdu_cb->pdulen = ntohs(cpl->len);
|
||||
|
||||
if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_HCRC_SHIFT)) {
|
||||
pr_info("tid 0x%x, status 0x%x, hcrc bad.\n", tid, status);
|
||||
pdu_cb->flags |= PDUCBF_RX_HCRC_ERR;
|
||||
}
|
||||
|
||||
if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_DCRC_SHIFT)) {
|
||||
pr_info("tid 0x%x, status 0x%x, dcrc bad.\n", tid, status);
|
||||
pdu_cb->flags |= PDUCBF_RX_DCRC_ERR;
|
||||
}
|
||||
|
||||
if (status & (1 << CPL_RX_ISCSI_DDP_STATUS_PAD_SHIFT))
|
||||
pr_info("tid 0x%x, status 0x%x, pad bad.\n", tid, status);
|
||||
|
||||
if ((status & (1 << CPL_RX_ISCSI_DDP_STATUS_DDP_SHIFT)) &&
|
||||
(!(pdu_cb->flags & PDUCBF_RX_DATA))) {
|
||||
pdu_cb->flags |= PDUCBF_RX_DATA_DDPD;
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
cxgbit_lro_add_packet_rsp(struct sk_buff *skb, u8 op, const __be64 *rsp)
|
||||
{
|
||||
struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
|
||||
struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb,
|
||||
lro_cb->pdu_idx);
|
||||
struct cpl_rx_iscsi_ddp *cpl = (struct cpl_rx_iscsi_ddp *)(rsp + 1);
|
||||
|
||||
cxgbit_proc_ddp_status(lro_cb->csk->tid, cpl, pdu_cb);
|
||||
|
||||
if (pdu_cb->flags & PDUCBF_RX_HDR)
|
||||
pdu_cb->complete = true;
|
||||
|
||||
lro_cb->complete = true;
|
||||
lro_cb->pdu_totallen += pdu_cb->pdulen;
|
||||
lro_cb->pdu_idx++;
|
||||
}
|
||||
|
||||
static void
|
||||
cxgbit_copy_frags(struct sk_buff *skb, const struct pkt_gl *gl,
|
||||
unsigned int offset)
|
||||
{
|
||||
u8 skb_frag_idx = skb_shinfo(skb)->nr_frags;
|
||||
u8 i;
|
||||
|
||||
/* usually there's just one frag */
|
||||
__skb_fill_page_desc(skb, skb_frag_idx, gl->frags[0].page,
|
||||
gl->frags[0].offset + offset,
|
||||
gl->frags[0].size - offset);
|
||||
for (i = 1; i < gl->nfrags; i++)
|
||||
__skb_fill_page_desc(skb, skb_frag_idx + i,
|
||||
gl->frags[i].page,
|
||||
gl->frags[i].offset,
|
||||
gl->frags[i].size);
|
||||
|
||||
skb_shinfo(skb)->nr_frags += gl->nfrags;
|
||||
|
||||
/* get a reference to the last page, we don't own it */
|
||||
get_page(gl->frags[gl->nfrags - 1].page);
|
||||
}
|
||||
|
||||
static void
|
||||
cxgbit_lro_add_packet_gl(struct sk_buff *skb, u8 op, const struct pkt_gl *gl)
|
||||
{
|
||||
struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
|
||||
struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb,
|
||||
lro_cb->pdu_idx);
|
||||
u32 len, offset;
|
||||
|
||||
if (op == CPL_ISCSI_HDR) {
|
||||
struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)gl->va;
|
||||
|
||||
offset = sizeof(struct cpl_iscsi_hdr);
|
||||
pdu_cb->flags |= PDUCBF_RX_HDR;
|
||||
pdu_cb->seq = ntohl(cpl->seq);
|
||||
len = ntohs(cpl->len);
|
||||
pdu_cb->hdr = gl->va + offset;
|
||||
pdu_cb->hlen = len;
|
||||
pdu_cb->hfrag_idx = skb_shinfo(skb)->nr_frags;
|
||||
|
||||
if (unlikely(gl->nfrags > 1))
|
||||
cxgbit_skcb_flags(skb) = 0;
|
||||
|
||||
lro_cb->complete = false;
|
||||
} else {
|
||||
struct cpl_iscsi_data *cpl = (struct cpl_iscsi_data *)gl->va;
|
||||
|
||||
offset = sizeof(struct cpl_iscsi_data);
|
||||
pdu_cb->flags |= PDUCBF_RX_DATA;
|
||||
len = ntohs(cpl->len);
|
||||
pdu_cb->dlen = len;
|
||||
pdu_cb->doffset = lro_cb->offset;
|
||||
pdu_cb->nr_dfrags = gl->nfrags;
|
||||
pdu_cb->dfrag_idx = skb_shinfo(skb)->nr_frags;
|
||||
}
|
||||
|
||||
cxgbit_copy_frags(skb, gl, offset);
|
||||
|
||||
pdu_cb->frags += gl->nfrags;
|
||||
lro_cb->offset += len;
|
||||
skb->len += len;
|
||||
skb->data_len += len;
|
||||
skb->truesize += len;
|
||||
}
|
||||
|
||||
static struct sk_buff *
|
||||
cxgbit_lro_init_skb(struct cxgbit_sock *csk, u8 op, const struct pkt_gl *gl,
|
||||
const __be64 *rsp, struct napi_struct *napi)
|
||||
{
|
||||
struct sk_buff *skb;
|
||||
struct cxgbit_lro_cb *lro_cb;
|
||||
|
||||
skb = napi_alloc_skb(napi, LRO_SKB_MAX_HEADROOM);
|
||||
|
||||
if (unlikely(!skb))
|
||||
return NULL;
|
||||
|
||||
memset(skb->data, 0, LRO_SKB_MAX_HEADROOM);
|
||||
|
||||
cxgbit_skcb_flags(skb) |= SKCBF_RX_LRO;
|
||||
|
||||
lro_cb = cxgbit_skb_lro_cb(skb);
|
||||
|
||||
cxgbit_get_csk(csk);
|
||||
|
||||
lro_cb->csk = csk;
|
||||
|
||||
return skb;
|
||||
}
|
||||
|
||||
static void cxgbit_queue_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
|
||||
{
|
||||
bool wakeup_thread = false;
|
||||
|
||||
spin_lock(&csk->rxq.lock);
|
||||
__skb_queue_tail(&csk->rxq, skb);
|
||||
if (skb_queue_len(&csk->rxq) == 1)
|
||||
wakeup_thread = true;
|
||||
spin_unlock(&csk->rxq.lock);
|
||||
|
||||
if (wakeup_thread)
|
||||
wake_up(&csk->waitq);
|
||||
}
|
||||
|
||||
static void cxgbit_lro_flush(struct t4_lro_mgr *lro_mgr, struct sk_buff *skb)
|
||||
{
|
||||
struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
|
||||
struct cxgbit_sock *csk = lro_cb->csk;
|
||||
|
||||
csk->lro_skb = NULL;
|
||||
|
||||
__skb_unlink(skb, &lro_mgr->lroq);
|
||||
cxgbit_queue_lro_skb(csk, skb);
|
||||
|
||||
cxgbit_put_csk(csk);
|
||||
|
||||
lro_mgr->lro_pkts++;
|
||||
lro_mgr->lro_session_cnt--;
|
||||
}
|
||||
|
||||
static void cxgbit_uld_lro_flush(struct t4_lro_mgr *lro_mgr)
|
||||
{
|
||||
struct sk_buff *skb;
|
||||
|
||||
while ((skb = skb_peek(&lro_mgr->lroq)))
|
||||
cxgbit_lro_flush(lro_mgr, skb);
|
||||
}
|
||||
|
||||
static int
|
||||
cxgbit_lro_receive(struct cxgbit_sock *csk, u8 op, const __be64 *rsp,
|
||||
const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr,
|
||||
struct napi_struct *napi)
|
||||
{
|
||||
struct sk_buff *skb;
|
||||
struct cxgbit_lro_cb *lro_cb;
|
||||
|
||||
if (!csk) {
|
||||
pr_err("%s: csk NULL, op 0x%x.\n", __func__, op);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (csk->lro_skb)
|
||||
goto add_packet;
|
||||
|
||||
start_lro:
|
||||
if (lro_mgr->lro_session_cnt >= MAX_LRO_SESSIONS) {
|
||||
cxgbit_uld_lro_flush(lro_mgr);
|
||||
goto start_lro;
|
||||
}
|
||||
|
||||
skb = cxgbit_lro_init_skb(csk, op, gl, rsp, napi);
|
||||
if (unlikely(!skb))
|
||||
goto out;
|
||||
|
||||
csk->lro_skb = skb;
|
||||
|
||||
__skb_queue_tail(&lro_mgr->lroq, skb);
|
||||
lro_mgr->lro_session_cnt++;
|
||||
|
||||
add_packet:
|
||||
skb = csk->lro_skb;
|
||||
lro_cb = cxgbit_skb_lro_cb(skb);
|
||||
|
||||
if ((gl && (((skb_shinfo(skb)->nr_frags + gl->nfrags) >
|
||||
MAX_SKB_FRAGS) || (lro_cb->pdu_totallen >= LRO_FLUSH_LEN_MAX))) ||
|
||||
(lro_cb->pdu_idx >= MAX_SKB_FRAGS)) {
|
||||
cxgbit_lro_flush(lro_mgr, skb);
|
||||
goto start_lro;
|
||||
}
|
||||
|
||||
if (gl)
|
||||
cxgbit_lro_add_packet_gl(skb, op, gl);
|
||||
else
|
||||
cxgbit_lro_add_packet_rsp(skb, op, rsp);
|
||||
|
||||
lro_mgr->lro_merged++;
|
||||
|
||||
return 0;
|
||||
|
||||
out:
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int
|
||||
cxgbit_uld_lro_rx_handler(void *hndl, const __be64 *rsp,
|
||||
const struct pkt_gl *gl, struct t4_lro_mgr *lro_mgr,
|
||||
struct napi_struct *napi)
|
||||
{
|
||||
struct cxgbit_device *cdev = hndl;
|
||||
struct cxgb4_lld_info *lldi = &cdev->lldi;
|
||||
struct cpl_tx_data *rpl = NULL;
|
||||
struct cxgbit_sock *csk = NULL;
|
||||
unsigned int tid = 0;
|
||||
struct sk_buff *skb;
|
||||
unsigned int op = *(u8 *)rsp;
|
||||
bool lro_flush = true;
|
||||
|
||||
switch (op) {
|
||||
case CPL_ISCSI_HDR:
|
||||
case CPL_ISCSI_DATA:
|
||||
case CPL_RX_ISCSI_DDP:
|
||||
case CPL_FW4_ACK:
|
||||
lro_flush = false;
|
||||
case CPL_ABORT_RPL_RSS:
|
||||
case CPL_PASS_ESTABLISH:
|
||||
case CPL_PEER_CLOSE:
|
||||
case CPL_CLOSE_CON_RPL:
|
||||
case CPL_ABORT_REQ_RSS:
|
||||
case CPL_SET_TCB_RPL:
|
||||
case CPL_RX_DATA:
|
||||
rpl = gl ? (struct cpl_tx_data *)gl->va :
|
||||
(struct cpl_tx_data *)(rsp + 1);
|
||||
tid = GET_TID(rpl);
|
||||
csk = lookup_tid(lldi->tids, tid);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (csk && csk->lro_skb && lro_flush)
|
||||
cxgbit_lro_flush(lro_mgr, csk->lro_skb);
|
||||
|
||||
if (!gl) {
|
||||
unsigned int len;
|
||||
|
||||
if (op == CPL_RX_ISCSI_DDP) {
|
||||
if (!cxgbit_lro_receive(csk, op, rsp, NULL, lro_mgr,
|
||||
napi))
|
||||
return 0;
|
||||
}
|
||||
|
||||
len = 64 - sizeof(struct rsp_ctrl) - 8;
|
||||
skb = napi_alloc_skb(napi, len);
|
||||
if (!skb)
|
||||
goto nomem;
|
||||
__skb_put(skb, len);
|
||||
skb_copy_to_linear_data(skb, &rsp[1], len);
|
||||
} else {
|
||||
if (unlikely(op != *(u8 *)gl->va)) {
|
||||
pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
|
||||
gl->va, be64_to_cpu(*rsp),
|
||||
be64_to_cpu(*(u64 *)gl->va),
|
||||
gl->tot_len);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (op == CPL_ISCSI_HDR || op == CPL_ISCSI_DATA) {
|
||||
if (!cxgbit_lro_receive(csk, op, rsp, gl, lro_mgr,
|
||||
napi))
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define RX_PULL_LEN 128
|
||||
skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
|
||||
if (unlikely(!skb))
|
||||
goto nomem;
|
||||
}
|
||||
|
||||
rpl = (struct cpl_tx_data *)skb->data;
|
||||
op = rpl->ot.opcode;
|
||||
cxgbit_skcb_rx_opcode(skb) = op;
|
||||
|
||||
pr_debug("cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
|
||||
cdev, op, rpl->ot.opcode_tid,
|
||||
ntohl(rpl->ot.opcode_tid), skb);
|
||||
|
||||
if (op < NUM_CPL_CMDS && cxgbit_cplhandlers[op]) {
|
||||
cxgbit_cplhandlers[op](cdev, skb);
|
||||
} else {
|
||||
pr_err("No handler for opcode 0x%x.\n", op);
|
||||
__kfree_skb(skb);
|
||||
}
|
||||
return 0;
|
||||
nomem:
|
||||
pr_err("%s OOM bailing out.\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CHELSIO_T4_DCB
|
||||
struct cxgbit_dcb_work {
|
||||
struct dcb_app_type dcb_app;
|
||||
struct work_struct work;
|
||||
};
|
||||
|
||||
static void
|
||||
cxgbit_update_dcb_priority(struct cxgbit_device *cdev, u8 port_id,
|
||||
u8 dcb_priority, u16 port_num)
|
||||
{
|
||||
struct cxgbit_sock *csk;
|
||||
struct sk_buff *skb;
|
||||
u16 local_port;
|
||||
bool wakeup_thread = false;
|
||||
|
||||
spin_lock_bh(&cdev->cskq.lock);
|
||||
list_for_each_entry(csk, &cdev->cskq.list, list) {
|
||||
if (csk->port_id != port_id)
|
||||
continue;
|
||||
|
||||
if (csk->com.local_addr.ss_family == AF_INET6) {
|
||||
struct sockaddr_in6 *sock_in6;
|
||||
|
||||
sock_in6 = (struct sockaddr_in6 *)&csk->com.local_addr;
|
||||
local_port = ntohs(sock_in6->sin6_port);
|
||||
} else {
|
||||
struct sockaddr_in *sock_in;
|
||||
|
||||
sock_in = (struct sockaddr_in *)&csk->com.local_addr;
|
||||
local_port = ntohs(sock_in->sin_port);
|
||||
}
|
||||
|
||||
if (local_port != port_num)
|
||||
continue;
|
||||
|
||||
if (csk->dcb_priority == dcb_priority)
|
||||
continue;
|
||||
|
||||
skb = alloc_skb(0, GFP_ATOMIC);
|
||||
if (!skb)
|
||||
continue;
|
||||
|
||||
spin_lock(&csk->rxq.lock);
|
||||
__skb_queue_tail(&csk->rxq, skb);
|
||||
if (skb_queue_len(&csk->rxq) == 1)
|
||||
wakeup_thread = true;
|
||||
spin_unlock(&csk->rxq.lock);
|
||||
|
||||
if (wakeup_thread) {
|
||||
wake_up(&csk->waitq);
|
||||
wakeup_thread = false;
|
||||
}
|
||||
}
|
||||
spin_unlock_bh(&cdev->cskq.lock);
|
||||
}
|
||||
|
||||
static void cxgbit_dcb_workfn(struct work_struct *work)
|
||||
{
|
||||
struct cxgbit_dcb_work *dcb_work;
|
||||
struct net_device *ndev;
|
||||
struct cxgbit_device *cdev = NULL;
|
||||
struct dcb_app_type *iscsi_app;
|
||||
u8 priority, port_id = 0xff;
|
||||
|
||||
dcb_work = container_of(work, struct cxgbit_dcb_work, work);
|
||||
iscsi_app = &dcb_work->dcb_app;
|
||||
|
||||
if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) {
|
||||
if (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY)
|
||||
goto out;
|
||||
|
||||
priority = iscsi_app->app.priority;
|
||||
|
||||
} else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) {
|
||||
if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM)
|
||||
goto out;
|
||||
|
||||
if (!iscsi_app->app.priority)
|
||||
goto out;
|
||||
|
||||
priority = ffs(iscsi_app->app.priority) - 1;
|
||||
} else {
|
||||
goto out;
|
||||
}
|
||||
|
||||
pr_debug("priority for ifid %d is %u\n",
|
||||
iscsi_app->ifindex, priority);
|
||||
|
||||
ndev = dev_get_by_index(&init_net, iscsi_app->ifindex);
|
||||
|
||||
if (!ndev)
|
||||
goto out;
|
||||
|
||||
mutex_lock(&cdev_list_lock);
|
||||
cdev = cxgbit_find_device(ndev, &port_id);
|
||||
|
||||
dev_put(ndev);
|
||||
|
||||
if (!cdev) {
|
||||
mutex_unlock(&cdev_list_lock);
|
||||
goto out;
|
||||
}
|
||||
|
||||
cxgbit_update_dcb_priority(cdev, port_id, priority,
|
||||
iscsi_app->app.protocol);
|
||||
mutex_unlock(&cdev_list_lock);
|
||||
out:
|
||||
kfree(dcb_work);
|
||||
}
|
||||
|
||||
static int
|
||||
cxgbit_dcbevent_notify(struct notifier_block *nb, unsigned long action,
|
||||
void *data)
|
||||
{
|
||||
struct cxgbit_dcb_work *dcb_work;
|
||||
struct dcb_app_type *dcb_app = data;
|
||||
|
||||
dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC);
|
||||
if (!dcb_work)
|
||||
return NOTIFY_DONE;
|
||||
|
||||
dcb_work->dcb_app = *dcb_app;
|
||||
INIT_WORK(&dcb_work->work, cxgbit_dcb_workfn);
|
||||
schedule_work(&dcb_work->work);
|
||||
return NOTIFY_OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
static enum target_prot_op cxgbit_get_sup_prot_ops(struct iscsi_conn *conn)
|
||||
{
|
||||
return TARGET_PROT_NORMAL;
|
||||
}
|
||||
|
||||
static struct iscsit_transport cxgbit_transport = {
|
||||
.name = DRV_NAME,
|
||||
.transport_type = ISCSI_HW_OFFLOAD,
|
||||
.priv_size = sizeof(struct cxgbit_cmd),
|
||||
.owner = THIS_MODULE,
|
||||
.iscsit_setup_np = cxgbit_setup_np,
|
||||
.iscsit_accept_np = cxgbit_accept_np,
|
||||
.iscsit_free_np = cxgbit_free_np,
|
||||
.iscsit_free_conn = cxgbit_free_conn,
|
||||
.iscsit_get_login_rx = cxgbit_get_login_rx,
|
||||
.iscsit_put_login_tx = cxgbit_put_login_tx,
|
||||
.iscsit_immediate_queue = iscsit_immediate_queue,
|
||||
.iscsit_response_queue = iscsit_response_queue,
|
||||
.iscsit_get_dataout = iscsit_build_r2ts_for_cmd,
|
||||
.iscsit_queue_data_in = iscsit_queue_rsp,
|
||||
.iscsit_queue_status = iscsit_queue_rsp,
|
||||
.iscsit_xmit_pdu = cxgbit_xmit_pdu,
|
||||
.iscsit_get_r2t_ttt = cxgbit_get_r2t_ttt,
|
||||
.iscsit_get_rx_pdu = cxgbit_get_rx_pdu,
|
||||
.iscsit_validate_params = cxgbit_validate_params,
|
||||
.iscsit_release_cmd = cxgbit_release_cmd,
|
||||
.iscsit_aborted_task = iscsit_aborted_task,
|
||||
.iscsit_get_sup_prot_ops = cxgbit_get_sup_prot_ops,
|
||||
};
|
||||
|
||||
static struct cxgb4_uld_info cxgbit_uld_info = {
|
||||
.name = DRV_NAME,
|
||||
.add = cxgbit_uld_add,
|
||||
.state_change = cxgbit_uld_state_change,
|
||||
.lro_rx_handler = cxgbit_uld_lro_rx_handler,
|
||||
.lro_flush = cxgbit_uld_lro_flush,
|
||||
};
|
||||
|
||||
#ifdef CONFIG_CHELSIO_T4_DCB
|
||||
static struct notifier_block cxgbit_dcbevent_nb = {
|
||||
.notifier_call = cxgbit_dcbevent_notify,
|
||||
};
|
||||
#endif
|
||||
|
||||
static int __init cxgbit_init(void)
|
||||
{
|
||||
cxgb4_register_uld(CXGB4_ULD_ISCSIT, &cxgbit_uld_info);
|
||||
iscsit_register_transport(&cxgbit_transport);
|
||||
|
||||
#ifdef CONFIG_CHELSIO_T4_DCB
|
||||
pr_info("%s dcb enabled.\n", DRV_NAME);
|
||||
register_dcbevent_notifier(&cxgbit_dcbevent_nb);
|
||||
#endif
|
||||
BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, cb) <
|
||||
sizeof(union cxgbit_skb_cb));
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __exit cxgbit_exit(void)
|
||||
{
|
||||
struct cxgbit_device *cdev, *tmp;
|
||||
|
||||
#ifdef CONFIG_CHELSIO_T4_DCB
|
||||
unregister_dcbevent_notifier(&cxgbit_dcbevent_nb);
|
||||
#endif
|
||||
mutex_lock(&cdev_list_lock);
|
||||
list_for_each_entry_safe(cdev, tmp, &cdev_list_head, list) {
|
||||
list_del(&cdev->list);
|
||||
cxgbit_put_cdev(cdev);
|
||||
}
|
||||
mutex_unlock(&cdev_list_lock);
|
||||
iscsit_unregister_transport(&cxgbit_transport);
|
||||
cxgb4_unregister_uld(CXGB4_ULD_ISCSIT);
|
||||
}
|
||||
|
||||
module_init(cxgbit_init);
|
||||
module_exit(cxgbit_exit);
|
||||
|
||||
MODULE_DESCRIPTION("Chelsio iSCSI target offload driver");
|
||||
MODULE_AUTHOR("Chelsio Communications");
|
||||
MODULE_VERSION(DRV_VERSION);
|
||||
MODULE_LICENSE("GPL");
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Загрузка…
Ссылка в новой задаче