321 строка
7.4 KiB
C
321 строка
7.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
|
|
#ifndef _IDXD_H_
|
|
#define _IDXD_H_
|
|
|
|
#include <linux/sbitmap.h>
|
|
#include <linux/dmaengine.h>
|
|
#include <linux/percpu-rwsem.h>
|
|
#include <linux/wait.h>
|
|
#include <linux/cdev.h>
|
|
#include "registers.h"
|
|
|
|
#define IDXD_DRIVER_VERSION "1.00"
|
|
|
|
extern struct kmem_cache *idxd_desc_pool;
|
|
|
|
#define IDXD_REG_TIMEOUT 50
|
|
#define IDXD_DRAIN_TIMEOUT 5000
|
|
|
|
enum idxd_type {
|
|
IDXD_TYPE_UNKNOWN = -1,
|
|
IDXD_TYPE_DSA = 0,
|
|
IDXD_TYPE_MAX
|
|
};
|
|
|
|
#define IDXD_NAME_SIZE 128
|
|
|
|
struct idxd_device_driver {
|
|
struct device_driver drv;
|
|
};
|
|
|
|
struct idxd_irq_entry {
|
|
struct idxd_device *idxd;
|
|
int id;
|
|
struct llist_head pending_llist;
|
|
struct list_head work_list;
|
|
};
|
|
|
|
struct idxd_group {
|
|
struct device conf_dev;
|
|
struct idxd_device *idxd;
|
|
struct grpcfg grpcfg;
|
|
int id;
|
|
int num_engines;
|
|
int num_wqs;
|
|
bool use_token_limit;
|
|
u8 tokens_allowed;
|
|
u8 tokens_reserved;
|
|
int tc_a;
|
|
int tc_b;
|
|
};
|
|
|
|
#define IDXD_MAX_PRIORITY 0xf
|
|
|
|
enum idxd_wq_state {
|
|
IDXD_WQ_DISABLED = 0,
|
|
IDXD_WQ_ENABLED,
|
|
};
|
|
|
|
enum idxd_wq_flag {
|
|
WQ_FLAG_DEDICATED = 0,
|
|
};
|
|
|
|
enum idxd_wq_type {
|
|
IDXD_WQT_NONE = 0,
|
|
IDXD_WQT_KERNEL,
|
|
IDXD_WQT_USER,
|
|
};
|
|
|
|
struct idxd_cdev {
|
|
struct cdev cdev;
|
|
struct device *dev;
|
|
int minor;
|
|
struct wait_queue_head err_queue;
|
|
};
|
|
|
|
#define IDXD_ALLOCATED_BATCH_SIZE 128U
|
|
#define WQ_NAME_SIZE 1024
|
|
#define WQ_TYPE_SIZE 10
|
|
|
|
enum idxd_op_type {
|
|
IDXD_OP_BLOCK = 0,
|
|
IDXD_OP_NONBLOCK = 1,
|
|
};
|
|
|
|
enum idxd_complete_type {
|
|
IDXD_COMPLETE_NORMAL = 0,
|
|
IDXD_COMPLETE_ABORT,
|
|
};
|
|
|
|
struct idxd_wq {
|
|
void __iomem *dportal;
|
|
struct device conf_dev;
|
|
struct idxd_cdev idxd_cdev;
|
|
struct idxd_device *idxd;
|
|
int id;
|
|
enum idxd_wq_type type;
|
|
struct idxd_group *group;
|
|
int client_count;
|
|
struct mutex wq_lock; /* mutex for workqueue */
|
|
u32 size;
|
|
u32 threshold;
|
|
u32 priority;
|
|
enum idxd_wq_state state;
|
|
unsigned long flags;
|
|
union wqcfg wqcfg;
|
|
u32 vec_ptr; /* interrupt steering */
|
|
struct dsa_hw_desc **hw_descs;
|
|
int num_descs;
|
|
struct dsa_completion_record *compls;
|
|
dma_addr_t compls_addr;
|
|
int compls_size;
|
|
struct idxd_desc **descs;
|
|
struct sbitmap_queue sbq;
|
|
struct dma_chan dma_chan;
|
|
char name[WQ_NAME_SIZE + 1];
|
|
};
|
|
|
|
struct idxd_engine {
|
|
struct device conf_dev;
|
|
int id;
|
|
struct idxd_group *group;
|
|
struct idxd_device *idxd;
|
|
};
|
|
|
|
/* shadow registers */
|
|
struct idxd_hw {
|
|
u32 version;
|
|
union gen_cap_reg gen_cap;
|
|
union wq_cap_reg wq_cap;
|
|
union group_cap_reg group_cap;
|
|
union engine_cap_reg engine_cap;
|
|
struct opcap opcap;
|
|
};
|
|
|
|
enum idxd_device_state {
|
|
IDXD_DEV_HALTED = -1,
|
|
IDXD_DEV_DISABLED = 0,
|
|
IDXD_DEV_CONF_READY,
|
|
IDXD_DEV_ENABLED,
|
|
};
|
|
|
|
enum idxd_device_flag {
|
|
IDXD_FLAG_CONFIGURABLE = 0,
|
|
IDXD_FLAG_CMD_RUNNING,
|
|
};
|
|
|
|
struct idxd_device {
|
|
enum idxd_type type;
|
|
struct device conf_dev;
|
|
struct list_head list;
|
|
struct idxd_hw hw;
|
|
enum idxd_device_state state;
|
|
unsigned long flags;
|
|
int id;
|
|
int major;
|
|
|
|
struct pci_dev *pdev;
|
|
void __iomem *reg_base;
|
|
|
|
spinlock_t dev_lock; /* spinlock for device */
|
|
struct completion *cmd_done;
|
|
struct idxd_group *groups;
|
|
struct idxd_wq *wqs;
|
|
struct idxd_engine *engines;
|
|
|
|
int num_groups;
|
|
|
|
u32 msix_perm_offset;
|
|
u32 wqcfg_offset;
|
|
u32 grpcfg_offset;
|
|
u32 perfmon_offset;
|
|
|
|
u64 max_xfer_bytes;
|
|
u32 max_batch_size;
|
|
int max_groups;
|
|
int max_engines;
|
|
int max_tokens;
|
|
int max_wqs;
|
|
int max_wq_size;
|
|
int token_limit;
|
|
int nr_tokens; /* non-reserved tokens */
|
|
|
|
union sw_err_reg sw_err;
|
|
wait_queue_head_t cmd_waitq;
|
|
struct msix_entry *msix_entries;
|
|
int num_wq_irqs;
|
|
struct idxd_irq_entry *irq_entries;
|
|
|
|
struct dma_device dma_dev;
|
|
struct workqueue_struct *wq;
|
|
struct work_struct work;
|
|
};
|
|
|
|
/* IDXD software descriptor */
|
|
struct idxd_desc {
|
|
struct dsa_hw_desc *hw;
|
|
dma_addr_t desc_dma;
|
|
struct dsa_completion_record *completion;
|
|
dma_addr_t compl_dma;
|
|
struct dma_async_tx_descriptor txd;
|
|
struct llist_node llnode;
|
|
struct list_head list;
|
|
int id;
|
|
int cpu;
|
|
struct idxd_wq *wq;
|
|
};
|
|
|
|
#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
|
|
#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
|
|
|
|
extern struct bus_type dsa_bus_type;
|
|
|
|
static inline bool wq_dedicated(struct idxd_wq *wq)
|
|
{
|
|
return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
|
|
}
|
|
|
|
enum idxd_portal_prot {
|
|
IDXD_PORTAL_UNLIMITED = 0,
|
|
IDXD_PORTAL_LIMITED,
|
|
};
|
|
|
|
static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
|
|
{
|
|
return prot * 0x1000;
|
|
}
|
|
|
|
static inline int idxd_get_wq_portal_full_offset(int wq_id,
|
|
enum idxd_portal_prot prot)
|
|
{
|
|
return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
|
|
}
|
|
|
|
static inline void idxd_set_type(struct idxd_device *idxd)
|
|
{
|
|
struct pci_dev *pdev = idxd->pdev;
|
|
|
|
if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
|
|
idxd->type = IDXD_TYPE_DSA;
|
|
else
|
|
idxd->type = IDXD_TYPE_UNKNOWN;
|
|
}
|
|
|
|
static inline void idxd_wq_get(struct idxd_wq *wq)
|
|
{
|
|
wq->client_count++;
|
|
}
|
|
|
|
static inline void idxd_wq_put(struct idxd_wq *wq)
|
|
{
|
|
wq->client_count--;
|
|
}
|
|
|
|
static inline int idxd_wq_refcount(struct idxd_wq *wq)
|
|
{
|
|
return wq->client_count;
|
|
};
|
|
|
|
const char *idxd_get_dev_name(struct idxd_device *idxd);
|
|
int idxd_register_bus_type(void);
|
|
void idxd_unregister_bus_type(void);
|
|
int idxd_setup_sysfs(struct idxd_device *idxd);
|
|
void idxd_cleanup_sysfs(struct idxd_device *idxd);
|
|
int idxd_register_driver(void);
|
|
void idxd_unregister_driver(void);
|
|
struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
|
|
|
|
/* device interrupt control */
|
|
irqreturn_t idxd_irq_handler(int vec, void *data);
|
|
irqreturn_t idxd_misc_thread(int vec, void *data);
|
|
irqreturn_t idxd_wq_thread(int irq, void *data);
|
|
void idxd_mask_error_interrupts(struct idxd_device *idxd);
|
|
void idxd_unmask_error_interrupts(struct idxd_device *idxd);
|
|
void idxd_mask_msix_vectors(struct idxd_device *idxd);
|
|
void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
|
|
void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
|
|
|
|
/* device control */
|
|
void idxd_device_init_reset(struct idxd_device *idxd);
|
|
int idxd_device_enable(struct idxd_device *idxd);
|
|
int idxd_device_disable(struct idxd_device *idxd);
|
|
void idxd_device_reset(struct idxd_device *idxd);
|
|
void idxd_device_cleanup(struct idxd_device *idxd);
|
|
int idxd_device_config(struct idxd_device *idxd);
|
|
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
|
|
|
|
/* work queue control */
|
|
int idxd_wq_alloc_resources(struct idxd_wq *wq);
|
|
void idxd_wq_free_resources(struct idxd_wq *wq);
|
|
int idxd_wq_enable(struct idxd_wq *wq);
|
|
int idxd_wq_disable(struct idxd_wq *wq);
|
|
void idxd_wq_drain(struct idxd_wq *wq);
|
|
int idxd_wq_map_portal(struct idxd_wq *wq);
|
|
void idxd_wq_unmap_portal(struct idxd_wq *wq);
|
|
void idxd_wq_disable_cleanup(struct idxd_wq *wq);
|
|
|
|
/* submission */
|
|
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
|
|
struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
|
|
void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
|
|
|
|
/* dmaengine */
|
|
int idxd_register_dma_device(struct idxd_device *idxd);
|
|
void idxd_unregister_dma_device(struct idxd_device *idxd);
|
|
int idxd_register_dma_channel(struct idxd_wq *wq);
|
|
void idxd_unregister_dma_channel(struct idxd_wq *wq);
|
|
void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
|
|
void idxd_dma_complete_txd(struct idxd_desc *desc,
|
|
enum idxd_complete_type comp_type);
|
|
dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx);
|
|
|
|
/* cdev */
|
|
int idxd_cdev_register(void);
|
|
void idxd_cdev_remove(void);
|
|
int idxd_cdev_get_major(struct idxd_device *idxd);
|
|
int idxd_wq_add_cdev(struct idxd_wq *wq);
|
|
void idxd_wq_del_cdev(struct idxd_wq *wq);
|
|
|
|
#endif
|