diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig index 4904097dfd490f..49bd56edcadce3 100644 --- a/drivers/nvme/target/Kconfig +++ b/drivers/nvme/target/Kconfig @@ -127,3 +127,12 @@ config NVME_TARGET_PCI_EPF capable PCI controller. If unsure, say N. + +config NVME_TARGET_DELAY_REQUESTS + bool "NVMe over Fabrics target request delay" + depends on NVME_TARGET && NVME_TARGET_DEBUGFS + help + This is a testing feature to allow delaying request completion in an + NVMe over Fabrics target, which allows for support of the cancel command. + + If unsure, say N. diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile index ed8522911d1f73..5d9848605285bc 100644 --- a/drivers/nvme/target/Makefile +++ b/drivers/nvme/target/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_NVME_TARGET_PCI_EPF) += nvmet-pci-epf.o nvmet-y += core.o configfs.o admin-cmd.o fabrics-cmd.o \ discovery.o io-cmd-file.o io-cmd-bdev.o pr.o +nvmet-$(CONFIG_NVME_TARGET_DELAY_REQUESTS) += io-cmd-cancel.o nvmet-$(CONFIG_NVME_TARGET_DEBUGFS) += debugfs.o nvmet-$(CONFIG_NVME_TARGET_PASSTHRU) += passthru.o nvmet-$(CONFIG_BLK_DEV_ZONED) += zns.o diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 3842087209da55..3cca4f4221922d 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -389,6 +389,9 @@ static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log) log->iocs[nvme_cmd_resv_release] = log->iocs[nvme_cmd_resv_report] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP); +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + log->iocs[nvme_cmd_cancel] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP); +#endif log->iocs[nvme_cmd_write] = log->iocs[nvme_cmd_write_zeroes] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC); diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index a87567f40c9155..c2150a3d9cf3a3 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include #include @@ -803,10 +805,23 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status) nvmet_put_namespace(ns); } +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +static void nvmet_delayed_execute_req(struct work_struct *work); +#endif + void nvmet_req_complete(struct nvmet_req *req, u16 status) { struct nvmet_sq *sq = req->sq; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + unsigned long flags; + /* only need to update the xarray if this was a delayed request */ + if (req->req_work.work.func == nvmet_delayed_execute_req) { + xa_lock_irqsave(&sq->outstanding_requests, flags); + __xa_erase(&sq->outstanding_requests, req->cmd->common.command_id); + xa_unlock_irqrestore(&sq->outstanding_requests, flags); + } +#endif __nvmet_req_complete(req, status); percpu_ref_put(&sq->ref); } @@ -1027,6 +1042,9 @@ int nvmet_sq_init(struct nvmet_sq *sq, struct nvmet_cq *cq) nvmet_auth_sq_init(sq); sq->cq = cq; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + xa_init_flags(&sq->outstanding_requests, XA_FLAGS_LOCK_IRQ); +#endif return 0; } EXPORT_SYMBOL_GPL(nvmet_sq_init); @@ -1105,6 +1123,14 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req) if (nvmet_is_passthru_req(req)) return nvmet_parse_passthru_io_cmd(req); +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + if (req->cmd->common.opcode == nvme_cmd_cancel) { + req->execute = nvmet_execute_cancel; + if (req->cmd->cancel.nsid == NVME_NSID_ALL) + return 0; + } +#endif + ret = nvmet_req_find_ns(req); if (unlikely(ret)) return ret; @@ -1792,6 +1818,46 @@ ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl, return ctrl->ops->host_traddr(ctrl, traddr, traddr_len); } +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +static void nvmet_delayed_execute_req(struct work_struct *work) { + struct nvmet_req *req = + container_of(to_delayed_work(work), struct nvmet_req, req_work); + req->execute(req); +} + +void nvmet_execute_request(struct nvmet_req *req) { + struct nvmet_ctrl *ctrl = req->sq->ctrl; + int delay_count; + u32 delay_msec; + unsigned long flags; + + if (unlikely(req->sq->qid == 0)) + return req->execute(req); + + if (ctrl) { + delay_count = atomic_dec_if_positive(&ctrl->delay_count) + 1; + delay_msec = ctrl->delay_msec; + } + if (!(ctrl && delay_count && delay_msec)) + return req->execute(req); + + xa_lock_irqsave(&req->sq->outstanding_requests, flags); + int ret = __xa_insert(&req->sq->outstanding_requests, + req->cmd->common.command_id, req, GFP_KERNEL); + xa_unlock_irqrestore(&req->sq->outstanding_requests, flags); + + if (ret) { + pr_err("nvmet: failure to delay command %d", + req->cmd->common.command_id); + return req->execute(req); + } + + INIT_DELAYED_WORK(&req->req_work, nvmet_delayed_execute_req); + queue_delayed_work(nvmet_wq, &req->req_work, msecs_to_jiffies(delay_msec)); +} +EXPORT_SYMBOL_GPL(nvmet_execute_request); +#endif + static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, const char *subsysnqn) { diff --git a/drivers/nvme/target/debugfs.c b/drivers/nvme/target/debugfs.c index 5dcbd5aa86e194..0571f86e07e5fe 100644 --- a/drivers/nvme/target/debugfs.c +++ b/drivers/nvme/target/debugfs.c @@ -153,6 +153,42 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p) NVMET_DEBUGFS_ATTR(nvmet_ctrl_tls_concat); #endif +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +static int nvmet_ctrl_delay_show(struct seq_file *m, void *p) +{ + struct nvmet_ctrl *ctrl = m->private; + int delay_count = atomic_read(&ctrl->delay_count); + + seq_printf(m, "%u %u\n", delay_count, ctrl->delay_msec); + return 0; +} + +static ssize_t nvmet_ctrl_delay_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = file->private_data; + struct nvmet_ctrl *ctrl = m->private; + char delay_buf[22] = {}; + int delay_count; + int delay_msec; + int n; + + if (count >= sizeof(delay_buf)) + return -EINVAL; + if (copy_from_user(delay_buf, buf, count)) + return -EFAULT; + + n = sscanf(delay_buf, "%u %u", &delay_count, &delay_msec); + if (n < 1 || n > 2) + return -EINVAL; + if (n == 2) + ctrl->delay_msec = delay_msec; + atomic_set(&ctrl->delay_count, delay_count); + return count; +} +NVMET_DEBUGFS_RW_ATTR(nvmet_ctrl_delay); +#endif /* CONFIG_NVME_TARGET_DELAY_REQUESTS */ + int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) { char name[32]; @@ -183,6 +219,10 @@ int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) &nvmet_ctrl_tls_concat_fops); debugfs_create_file("tls_key", S_IRUSR, ctrl->debugfs_dir, ctrl, &nvmet_ctrl_tls_key_fops); +#endif +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + debugfs_create_file("delay", S_IWUSR, ctrl->debugfs_dir, ctrl, + &nvmet_ctrl_delay_fops); #endif return 0; } diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index d161707559ce69..36cc57005969d8 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -2390,7 +2390,7 @@ nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod) } /* data transfer complete, resume with nvmet layer */ - fod->req.execute(&fod->req); + nvmet_execute_request(&fod->req); break; case NVMET_FCOP_READDATA: @@ -2599,7 +2599,7 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, * can invoke the nvmet_layer now. If read data, cmd completion will * push the data */ - fod->req.execute(&fod->req); + nvmet_execute_request(&fod->req); return; transport_error: diff --git a/drivers/nvme/target/io-cmd-cancel.c b/drivers/nvme/target/io-cmd-cancel.c new file mode 100644 index 00000000000000..8d7f363c916376 --- /dev/null +++ b/drivers/nvme/target/io-cmd-cancel.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NVMe I/O cancel command implementation. + * Copyright (c) 2023 Red Hat + */ + +#include "nvmet.h" + +void nvmet_execute_cancel(struct nvmet_req *req) +{ + u16 cid; + __le16 sqid; + bool mult_cmds; + int ret = 0; + struct nvmet_ctrl *ctrl = req->sq->ctrl; + struct nvmet_sq *sq = req->sq; + struct nvmet_req *treq; + u32 canceled = 0; + + if (!nvmet_check_transfer_len(req, 0)) + return; + + sqid = le16_to_cpu(req->cmd->cancel.sqid); + if (sqid > ctrl->subsys->max_qid) { + ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto exit; + } + + mult_cmds = req->cmd->cancel.action & NVME_CANCEL_ACTION_MUL_CMD; + cid = req->cmd->cancel.cid; + + if (cid == req->cmd->cancel.command_id && !mult_cmds) { + /* If action is set to "single command" and cid is + * set to the cid of this cancel command, then + * the controller shall abort the command with + * an "invalid cid" status code. + */ + ret = NVME_SC_INVALID_CID | NVME_STATUS_DNR; + } else if ((cid != 0xFFFF && mult_cmds) || sqid != sq->qid) { + /* if action is set to "multiple commands" and + * cid isn't set to 0xFFFF, then abort the command + * with an "invalid field" status. + * if the sqid field doesn't match the sqid of + * the queue to which the cancel command is submitted, + * then abort the command with an "invalid field" status. + */ + ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + } + + if (!mult_cmds) { + treq = xa_load(&sq->outstanding_requests, cid); + if (treq) { + if (cancel_delayed_work(&treq->req_work)) { + pr_info("nvmet: CANCEL success: %d", cid); + nvmet_req_complete(treq, NVME_SC_ABORT_REQ); + canceled += 1; + } else { + pr_info("nvmet: CANCEL failed: %d", cid); + } + } else { + pr_info("nvmet: CANCEL request not found: %d", cid); + } + } else { + unsigned long ucid; + xa_for_each(&sq->outstanding_requests, ucid, treq) { + if (cancel_delayed_work(&treq->req_work)) { + nvmet_req_complete(treq, NVME_SC_ABORT_REQ); + canceled += 1; + } + } + pr_info("nvmet: CANCEL removed %d requests", canceled); + } +exit: + nvmet_set_result(req, canceled); + nvmet_req_complete(req, ret); +} + diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index c9cf0ddc1b8563..e521d5810150d0 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -127,7 +127,7 @@ static void nvme_loop_execute_work(struct work_struct *work) struct nvme_loop_iod *iod = container_of(work, struct nvme_loop_iod, work); - iod->req.execute(&iod->req); + nvmet_execute_request(&iod->req); } static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 64af6bf569bf45..ef2354fa4a3c38 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -174,6 +174,9 @@ struct nvmet_sq { #endif struct completion free_done; struct completion confirm_done; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + struct xarray outstanding_requests; +#endif }; struct nvmet_ana_group { @@ -309,6 +312,10 @@ struct nvmet_ctrl { #endif #ifdef CONFIG_NVME_TARGET_TCP_TLS struct key *tls_key; +#endif +#ifdef CONFIG_NVME_TARGET_DELAY_REQUESTS + atomic_t delay_count; + u32 delay_msec; #endif struct nvmet_pr_log_mgr pr_log_mgr; }; @@ -495,6 +502,9 @@ struct nvmet_req { u16 error_loc; u64 error_slba; struct nvmet_pr_per_ctrl_ref *pc_ref; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + struct delayed_work req_work; +#endif }; #define NVMET_MAX_MPOOL_BVEC 16 @@ -571,6 +581,7 @@ size_t nvmet_req_transfer_len(struct nvmet_req *req); bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len); bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len); void nvmet_req_complete(struct nvmet_req *req, u16 status); +void nvmet_req_complete_delayed(struct nvmet_req *req, u16 status); int nvmet_req_alloc_sgls(struct nvmet_req *req); void nvmet_req_free_sgls(struct nvmet_req *req); @@ -727,6 +738,8 @@ void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req); void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req); void nvmet_bdev_execute_zone_append(struct nvmet_req *req); +void nvmet_execute_cancel(struct nvmet_req *req); + static inline u32 nvmet_rw_data_len(struct nvmet_req *req) { return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) << @@ -989,4 +1002,10 @@ struct nvmet_feat_arbitration { u8 ab; }; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +void nvmet_execute_request(struct nvmet_req *req); +#else +static inline void nvmet_execute_request(struct nvmet_req *req) { req->execute(req); } +#endif + #endif /* _NVMET_H */ diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 2d6eb89f98af2d..c9cb70e30788b2 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -772,7 +772,7 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc) if (unlikely(status)) nvmet_req_complete(&rsp->req, status); else - rsp->req.execute(&rsp->req); + nvmet_execute_request(&rsp->req); } static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc) @@ -957,7 +957,7 @@ static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp) queue->cm_id->port_num, &rsp->read_cqe, NULL)) nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR); } else { - rsp->req.execute(&rsp->req); + nvmet_execute_request(&rsp->req); } return true; diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 164a564ba3b4e9..ef7cdc96ad57e0 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -622,7 +622,7 @@ static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd) if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED)) nvmet_tcp_queue_response(&cmd->req); else - cmd->req.execute(&cmd->req); + nvmet_execute_request(&cmd->req); } static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd) @@ -1116,7 +1116,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) goto out; } - queue->cmd->req.execute(&queue->cmd->req); + nvmet_execute_request(&queue->cmd->req); out: nvmet_prepare_receive_pdu(queue); return ret;