Nvme cancel lsfmm 2

drivers/nvme/target/Kconfig

@@ -127,3 +127,12 @@ config NVME_TARGET_PCI_EPF
 	  capable PCI controller.
 
 	  If unsure, say N.
+
+config NVME_TARGET_DELAY_REQUESTS
+	bool "NVMe over Fabrics target request delay"
+	depends on NVME_TARGET && NVME_TARGET_DEBUGFS
+	help
+	  This is a testing feature to allow delaying request completion in an
+	  NVMe over Fabrics target, which allows for support of the cancel command.
+
+	  If unsure, say N.

drivers/nvme/target/Makefile

@@ -12,6 +12,7 @@ obj-$(CONFIG_NVME_TARGET_PCI_EPF)	+= nvmet-pci-epf.o
 
 nvmet-y		+= core.o configfs.o admin-cmd.o fabrics-cmd.o \
 			discovery.o io-cmd-file.o io-cmd-bdev.o pr.o
+nvmet-$(CONFIG_NVME_TARGET_DELAY_REQUESTS) += io-cmd-cancel.o
 nvmet-$(CONFIG_NVME_TARGET_DEBUGFS)	+= debugfs.o
 nvmet-$(CONFIG_NVME_TARGET_PASSTHRU)	+= passthru.o
 nvmet-$(CONFIG_BLK_DEV_ZONED)		+= zns.o

drivers/nvme/target/admin-cmd.c

@@ -389,6 +389,9 @@ static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log)
 	log->iocs[nvme_cmd_resv_release] =
 	log->iocs[nvme_cmd_resv_report] =
 		cpu_to_le32(NVME_CMD_EFFECTS_CSUPP);
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	log->iocs[nvme_cmd_cancel] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP);
+#endif
 	log->iocs[nvme_cmd_write] =
 	log->iocs[nvme_cmd_write_zeroes] =
 		cpu_to_le32(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC);

drivers/nvme/target/core.c

@@ -7,7 +7,9 @@
 #include <linux/hex.h>
 #include <linux/module.h>
 #include <linux/random.h>
+#include <linux/jiffies.h>
 #include <linux/rculist.h>
+#include <linux/workqueue.h>
 #include <linux/pci-p2pdma.h>
 #include <linux/scatterlist.h>
 
@@ -803,10 +805,23 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
 		nvmet_put_namespace(ns);
 }
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+static void nvmet_delayed_execute_req(struct work_struct *work);
+#endif
+
 void nvmet_req_complete(struct nvmet_req *req, u16 status)
 {
 	struct nvmet_sq *sq = req->sq;
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	unsigned long flags;
 
+	/* only need to update the xarray if this was a delayed request */
+	if (req->req_work.work.func == nvmet_delayed_execute_req) {
+		xa_lock_irqsave(&sq->outstanding_requests, flags);
+		__xa_erase(&sq->outstanding_requests, req->cmd->common.command_id);
+		xa_unlock_irqrestore(&sq->outstanding_requests, flags);
+	}
+#endif
 	__nvmet_req_complete(req, status);
 	percpu_ref_put(&sq->ref);
 }
@@ -1027,6 +1042,9 @@ int nvmet_sq_init(struct nvmet_sq *sq, struct nvmet_cq *cq)
 	nvmet_auth_sq_init(sq);
 	sq->cq = cq;
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	xa_init_flags(&sq->outstanding_requests, XA_FLAGS_LOCK_IRQ);
+#endif
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nvmet_sq_init);
@@ -1105,6 +1123,14 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
 	if (nvmet_is_passthru_req(req))
 		return nvmet_parse_passthru_io_cmd(req);
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	if (req->cmd->common.opcode == nvme_cmd_cancel) {
+		req->execute = nvmet_execute_cancel;
+		if (req->cmd->cancel.nsid == NVME_NSID_ALL)
+			return 0;
+	}
+#endif
+
 	ret = nvmet_req_find_ns(req);
 	if (unlikely(ret))
 		return ret;
@@ -1792,6 +1818,46 @@ ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl,
 	return ctrl->ops->host_traddr(ctrl, traddr, traddr_len);
 }
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+static void nvmet_delayed_execute_req(struct work_struct *work) {
+	struct nvmet_req *req =
+		container_of(to_delayed_work(work), struct nvmet_req, req_work);
+	req->execute(req);
+}
+
+void nvmet_execute_request(struct nvmet_req *req) {
+	struct nvmet_ctrl *ctrl = req->sq->ctrl;
+	int delay_count;
+	u32 delay_msec;
+	unsigned long flags;
+
+	if (unlikely(req->sq->qid == 0))
+		return req->execute(req);
+
+	if (ctrl) {
+		delay_count = atomic_dec_if_positive(&ctrl->delay_count) + 1;
+		delay_msec = ctrl->delay_msec;
+	}
+	if (!(ctrl && delay_count && delay_msec))
+		return req->execute(req);
+
+	xa_lock_irqsave(&req->sq->outstanding_requests, flags);
+	int ret = __xa_insert(&req->sq->outstanding_requests,
+			    req->cmd->common.command_id, req, GFP_KERNEL);
+	xa_unlock_irqrestore(&req->sq->outstanding_requests, flags);
+
+	if (ret) {
+		pr_err("nvmet: failure to delay command %d",
+		       req->cmd->common.command_id);
+		return req->execute(req);
+	}
+
+	INIT_DELAYED_WORK(&req->req_work, nvmet_delayed_execute_req);
+	queue_delayed_work(nvmet_wq, &req->req_work, msecs_to_jiffies(delay_msec));
+}
+EXPORT_SYMBOL_GPL(nvmet_execute_request);
+#endif
+
 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port,
 		const char *subsysnqn)
 {

drivers/nvme/target/debugfs.c

@@ -153,6 +153,42 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p)
 NVMET_DEBUGFS_ATTR(nvmet_ctrl_tls_concat);
 #endif
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+static int nvmet_ctrl_delay_show(struct seq_file *m, void *p)
+{
+	struct nvmet_ctrl *ctrl = m->private;
+	int delay_count = atomic_read(&ctrl->delay_count);
+
+	seq_printf(m, "%u %u\n", delay_count, ctrl->delay_msec);
+	return 0;
+}
+
+static ssize_t nvmet_ctrl_delay_write(struct file *file, const char __user *buf,
+				      size_t count, loff_t *ppos)
+{
+	struct seq_file *m = file->private_data;
+	struct nvmet_ctrl *ctrl = m->private;
+	char delay_buf[22] = {};
+	int delay_count;
+	int delay_msec;
+	int n;
+
+	if (count >= sizeof(delay_buf))
+		return -EINVAL;
+	if (copy_from_user(delay_buf, buf, count))
+		return -EFAULT;
+
+	n = sscanf(delay_buf, "%u %u", &delay_count, &delay_msec);
+	if (n < 1 || n > 2)
+		return -EINVAL;
+	if (n == 2)
+		ctrl->delay_msec = delay_msec;
+	atomic_set(&ctrl->delay_count, delay_count);
+	return count;
+}
+NVMET_DEBUGFS_RW_ATTR(nvmet_ctrl_delay);
+#endif /* CONFIG_NVME_TARGET_DELAY_REQUESTS */
+
 int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl)
 {
 	char name[32];
@@ -183,6 +219,10 @@ int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl)
 			    &nvmet_ctrl_tls_concat_fops);
 	debugfs_create_file("tls_key", S_IRUSR, ctrl->debugfs_dir, ctrl,
 			    &nvmet_ctrl_tls_key_fops);
+#endif
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	debugfs_create_file("delay", S_IWUSR, ctrl->debugfs_dir, ctrl,
+			    &nvmet_ctrl_delay_fops);
 #endif
 	return 0;
 }

drivers/nvme/target/fc.c

@@ -2390,7 +2390,7 @@ nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
 		}
 
 		/* data transfer complete, resume with nvmet layer */
-		fod->req.execute(&fod->req);
+		nvmet_execute_request(&fod->req);
 		break;
 
 	case NVMET_FCOP_READDATA:
@@ -2599,7 +2599,7 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
 	 * can invoke the nvmet_layer now. If read data, cmd completion will
 	 * push the data
 	 */
-	fod->req.execute(&fod->req);
+	nvmet_execute_request(&fod->req);
 	return;
 
 transport_error:

drivers/nvme/target/io-cmd-cancel.c

@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NVMe I/O cancel command implementation.
+ * Copyright (c) 2023 Red Hat
+ */
+
+#include "nvmet.h"
+
+void nvmet_execute_cancel(struct nvmet_req *req)
+{
+	u16 cid;
+	__le16 sqid;
+	bool mult_cmds;
+	int ret = 0;
+	struct nvmet_ctrl *ctrl = req->sq->ctrl;
+	struct nvmet_sq *sq = req->sq;
+	struct nvmet_req *treq;
+	u32 canceled = 0;
+
+	if (!nvmet_check_transfer_len(req, 0))
+		return;
+
+	sqid = le16_to_cpu(req->cmd->cancel.sqid);
+	if (sqid > ctrl->subsys->max_qid) {
+		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
+		goto exit;
+	}
+
+	mult_cmds = req->cmd->cancel.action & NVME_CANCEL_ACTION_MUL_CMD;
+	cid  = req->cmd->cancel.cid;
+
+	if (cid == req->cmd->cancel.command_id && !mult_cmds) {
+		/* If action is set to "single command" and cid is
+		 * set to the cid of this cancel command, then
+		 * the controller shall abort the command with
+		 * an "invalid cid" status code.
+		 */
+		ret = NVME_SC_INVALID_CID | NVME_STATUS_DNR;
+	} else if ((cid != 0xFFFF && mult_cmds) || sqid != sq->qid) {
+		/* if action is set to "multiple commands" and
+		 * cid isn't set to 0xFFFF, then abort the command
+		 * with an "invalid field" status.
+		 * if the sqid field doesn't match the sqid of
+		 * the queue to which the cancel command is submitted,
+		 * then abort the command with an "invalid field" status.
+		 */
+		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
+	}
+
+	if (!mult_cmds) {
+		treq = xa_load(&sq->outstanding_requests, cid);
+		if (treq) {
+			if (cancel_delayed_work(&treq->req_work)) {
+				pr_info("nvmet: CANCEL success: %d", cid);
+				nvmet_req_complete(treq, NVME_SC_ABORT_REQ);
+				canceled += 1;
+			} else {
+				pr_info("nvmet: CANCEL failed: %d", cid);
+			}
+		} else {
+			pr_info("nvmet: CANCEL request not found: %d", cid);
+		}
+	} else {
+		unsigned long ucid;
+		xa_for_each(&sq->outstanding_requests, ucid, treq) {
+			if (cancel_delayed_work(&treq->req_work)) {
+				nvmet_req_complete(treq, NVME_SC_ABORT_REQ);
+				canceled += 1;
+			}
+		}
+		pr_info("nvmet: CANCEL removed %d requests", canceled);
+	}
+exit:
+	nvmet_set_result(req, canceled);
+	nvmet_req_complete(req, ret);
+}
+

drivers/nvme/target/loop.c

@@ -127,7 +127,7 @@ static void nvme_loop_execute_work(struct work_struct *work)
 	struct nvme_loop_iod *iod =
 		container_of(work, struct nvme_loop_iod, work);
 
-	iod->req.execute(&iod->req);
+	nvmet_execute_request(&iod->req);
 }
 
 static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,

drivers/nvme/target/nvmet.h

@@ -174,6 +174,9 @@ struct nvmet_sq {
 #endif
 	struct completion	free_done;
 	struct completion	confirm_done;
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	struct xarray		outstanding_requests;
+#endif
 };
 
 struct nvmet_ana_group {
@@ -309,6 +312,10 @@ struct nvmet_ctrl {
 #endif
 #ifdef CONFIG_NVME_TARGET_TCP_TLS
 	struct key		*tls_key;
+#endif
+#ifdef CONFIG_NVME_TARGET_DELAY_REQUESTS
+	atomic_t		delay_count;
+	u32			delay_msec;
 #endif
 	struct nvmet_pr_log_mgr pr_log_mgr;
 };
@@ -495,6 +502,9 @@ struct nvmet_req {
 	u16			error_loc;
 	u64			error_slba;
 	struct nvmet_pr_per_ctrl_ref *pc_ref;
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+	struct delayed_work	req_work;
+#endif
 };
 
 #define NVMET_MAX_MPOOL_BVEC		16
@@ -571,6 +581,7 @@ size_t nvmet_req_transfer_len(struct nvmet_req *req);
 bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len);
 bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len);
 void nvmet_req_complete(struct nvmet_req *req, u16 status);
+void nvmet_req_complete_delayed(struct nvmet_req *req, u16 status);
 int nvmet_req_alloc_sgls(struct nvmet_req *req);
 void nvmet_req_free_sgls(struct nvmet_req *req);
 
@@ -727,6 +738,8 @@ void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req);
 void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req);
 void nvmet_bdev_execute_zone_append(struct nvmet_req *req);
 
+void nvmet_execute_cancel(struct nvmet_req *req);
+
 static inline u32 nvmet_rw_data_len(struct nvmet_req *req)
 {
 	return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) <<
@@ -989,4 +1002,10 @@ struct nvmet_feat_arbitration {
 	u8		ab;
 };
 
+#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS)
+void nvmet_execute_request(struct nvmet_req *req);
+#else
+static inline void nvmet_execute_request(struct nvmet_req *req) { req->execute(req); }
+#endif
+
 #endif /* _NVMET_H */

drivers/nvme/target/rdma.c

@@ -772,7 +772,7 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc)
 	if (unlikely(status))
 		nvmet_req_complete(&rsp->req, status);
 	else
-		rsp->req.execute(&rsp->req);
+		nvmet_execute_request(&rsp->req);
 }
 
 static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc)
@@ -957,7 +957,7 @@ static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp)
 				queue->cm_id->port_num, &rsp->read_cqe, NULL))
 			nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR);
 	} else {
-		rsp->req.execute(&rsp->req);
+		nvmet_execute_request(&rsp->req);
 	}
 
 	return true;

drivers/nvme/target/tcp.c

@@ -622,7 +622,7 @@ static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd)
 	if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED))
 		nvmet_tcp_queue_response(&cmd->req);
 	else
-		cmd->req.execute(&cmd->req);
+		nvmet_execute_request(&cmd->req);
 }
 
 static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd)
@@ -1116,7 +1116,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue)
 		goto out;
 	}
 
-	queue->cmd->req.execute(&queue->cmd->req);
+	nvmet_execute_request(&queue->cmd->req);
 out:
 	nvmet_prepare_receive_pdu(queue);
 	return ret;