diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 423c9c628e7bfa..e77c4740810218 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -858,7 +858,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) * doing a safe shutdown. */ if (!dead && shutdown && freeze) - nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT); + nvme_wait_freeze_timeout(&anv->ctrl); nvme_quiesce_io_queues(&anv->ctrl); diff --git a/drivers/nvme/host/constants.c b/drivers/nvme/host/constants.c index dc90df9e13a213..0fb6975a01df0e 100644 --- a/drivers/nvme/host/constants.c +++ b/drivers/nvme/host/constants.c @@ -19,6 +19,7 @@ static const char * const nvme_ops[] = { [nvme_cmd_resv_report] = "Reservation Report", [nvme_cmd_resv_acquire] = "Reservation Acquire", [nvme_cmd_resv_release] = "Reservation Release", + [nvme_cmd_cancel] = "Cancel", [nvme_cmd_zone_mgmt_send] = "Zone Management Send", [nvme_cmd_zone_mgmt_recv] = "Zone Management Receive", [nvme_cmd_zone_append] = "Zone Append", @@ -46,6 +47,7 @@ static const char * const nvme_admin_ops[] = { [nvme_admin_virtual_mgmt] = "Virtual Management", [nvme_admin_nvme_mi_send] = "NVMe Send MI", [nvme_admin_nvme_mi_recv] = "NVMe Receive MI", + [nvme_admin_cross_ctrl_reset] = "Cross Controller Reset", [nvme_admin_dbbuf] = "Doorbell Buffer Config", [nvme_admin_format_nvm] = "Format NVM", [nvme_admin_security_send] = "Security Send", diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0d623476e36f87..d9731f19b22086 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -554,6 +554,150 @@ void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl) } EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset); +static struct nvme_ctrl *nvme_find_ctrl_ccr(struct nvme_ctrl *ictrl, + u32 min_cntlid) +{ + struct nvme_subsystem *subsys = ictrl->subsys; + struct nvme_ctrl *ctrl, *sctrl = NULL; + unsigned long flags; + + mutex_lock(&nvme_subsystems_lock); + list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { + if (ctrl->cntlid < min_cntlid) + continue; + + if (atomic_dec_if_positive(&ctrl->ccr_limit) < 0) + continue; + + spin_lock_irqsave(&ctrl->lock, flags); + if (ctrl->state != NVME_CTRL_LIVE) { + spin_unlock_irqrestore(&ctrl->lock, flags); + atomic_inc(&ctrl->ccr_limit); + continue; + } + + /* + * We got a good candidate source controller that is locked and + * LIVE. However, no guarantee ctrl will not be deleted after + * ctrl->lock is released. Get a ref of both ctrl and admin_q + * so they do not disappear until we are done with them. + */ + WARN_ON_ONCE(!blk_get_queue(ctrl->admin_q)); + nvme_get_ctrl(ctrl); + spin_unlock_irqrestore(&ctrl->lock, flags); + sctrl = ctrl; + break; + } + mutex_unlock(&nvme_subsystems_lock); + return sctrl; +} + +static void nvme_put_ctrl_ccr(struct nvme_ctrl *sctrl) +{ + atomic_inc(&sctrl->ccr_limit); + blk_put_queue(sctrl->admin_q); + nvme_put_ctrl(sctrl); +} + +static int nvme_issue_wait_ccr(struct nvme_ctrl *sctrl, struct nvme_ctrl *ictrl, + unsigned long deadline) +{ + struct nvme_ccr_entry ccr = { }; + union nvme_result res = { 0 }; + struct nvme_command c = { }; + unsigned long flags, now, tmo = 0; + bool completed = false; + int ret = 0; + u32 result; + + init_completion(&ccr.complete); + ccr.ictrl = ictrl; + + spin_lock_irqsave(&sctrl->lock, flags); + list_add_tail(&ccr.list, &sctrl->ccr_list); + spin_unlock_irqrestore(&sctrl->lock, flags); + + c.ccr.opcode = nvme_admin_cross_ctrl_reset; + c.ccr.ciu = ictrl->ciu; + c.ccr.icid = cpu_to_le16(ictrl->cntlid); + c.ccr.cirn = cpu_to_le64(ictrl->cirn); + ret = __nvme_submit_sync_cmd(sctrl->admin_q, &c, &res, + NULL, 0, NVME_QID_ANY, 0); + if (ret) { + ret = -EIO; + goto out; + } + + result = le32_to_cpu(res.u32); + if (result & 0x01) /* Immediate Reset Successful */ + goto out; + + now = jiffies; + if (time_before(now, deadline)) + tmo = min_t(unsigned long, + secs_to_jiffies(ictrl->kato), deadline - now); + + if (!wait_for_completion_timeout(&ccr.complete, tmo)) { + ret = -ETIMEDOUT; + goto out; + } + + completed = true; + +out: + spin_lock_irqsave(&sctrl->lock, flags); + list_del(&ccr.list); + spin_unlock_irqrestore(&sctrl->lock, flags); + if (completed) { + if (ccr.ccrs == NVME_CCR_STATUS_SUCCESS) + return 0; + return -EREMOTEIO; + } + return ret; +} + +int nvme_fence_ctrl(struct nvme_ctrl *ictrl) +{ + unsigned long deadline, timeout; + struct nvme_ctrl *sctrl; + u32 min_cntlid = 0; + int ret; + + timeout = nvme_fence_timeout_ms(ictrl); + dev_info(ictrl->device, "attempting CCR, timeout %lums\n", timeout); + + deadline = jiffies + msecs_to_jiffies(timeout); + while (time_is_after_jiffies(deadline)) { + sctrl = nvme_find_ctrl_ccr(ictrl, min_cntlid); + if (!sctrl) { + dev_dbg(ictrl->device, + "failed to find source controller\n"); + return -EIO; + } + + ret = nvme_issue_wait_ccr(sctrl, ictrl, deadline); + if (!ret) { + dev_info(ictrl->device, "CCR succeeded using %s\n", + dev_name(sctrl->device)); + nvme_put_ctrl_ccr(sctrl); + return 0; + } + + min_cntlid = sctrl->cntlid + 1; + nvme_put_ctrl_ccr(sctrl); + + if (ret == -EIO) /* CCR command failed */ + continue; + + /* CCR operation failed or timed out */ + return ret; + } + + dev_info(ictrl->device, "CCR operation timeout\n"); + return -ETIMEDOUT; +} +EXPORT_SYMBOL_GPL(nvme_fence_ctrl); + bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, enum nvme_ctrl_state new_state) { @@ -574,10 +718,29 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, break; } break; + case NVME_CTRL_FENCING: + switch (old_state) { + case NVME_CTRL_LIVE: + changed = true; + fallthrough; + default: + break; + } + break; + case NVME_CTRL_FENCED: + switch (old_state) { + case NVME_CTRL_FENCING: + changed = true; + fallthrough; + default: + break; + } + break; case NVME_CTRL_RESETTING: switch (old_state) { case NVME_CTRL_NEW: case NVME_CTRL_LIVE: + case NVME_CTRL_FENCED: changed = true; fallthrough; default: @@ -724,10 +887,8 @@ void nvme_init_request(struct request *req, struct nvme_command *cmd) struct nvme_ns *ns = req->q->disk->private_data; logging_enabled = ns->head->passthru_err_log_enabled; - req->timeout = NVME_IO_TIMEOUT; } else { /* no queuedata implies admin queue */ logging_enabled = nr->ctrl->passthru_err_log_enabled; - req->timeout = NVME_ADMIN_TIMEOUT; } if (!logging_enabled) @@ -760,6 +921,8 @@ blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl, if (state != NVME_CTRL_DELETING_NOIO && state != NVME_CTRL_DELETING && + state != NVME_CTRL_FENCING && + state != NVME_CTRL_FENCED && state != NVME_CTRL_DEAD && !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) && !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH)) @@ -802,10 +965,12 @@ bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq, req->cmd->fabrics.fctype == nvme_fabrics_type_auth_receive)) return true; break; - default: - break; + case NVME_CTRL_FENCING: + case NVME_CTRL_FENCED: case NVME_CTRL_DEAD: return false; + default: + break; } } @@ -1232,6 +1397,20 @@ u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) } EXPORT_SYMBOL_NS_GPL(nvme_command_effects, "NVME_TARGET_PASSTHRU"); +bool nvme_io_command_supported(struct nvme_ctrl *ctrl, u8 opcode) +{ + u32 effects = le32_to_cpu(ctrl->effects->iocs[opcode]); + + return effects & NVME_CMD_EFFECTS_CSUPP; +} +EXPORT_SYMBOL_GPL(nvme_io_command_supported); + +bool nvme_is_cancel(struct nvme_command *cmd) +{ + return cmd->common.opcode == nvme_cmd_cancel; +} +EXPORT_SYMBOL_GPL(nvme_is_cancel); + u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) { u32 effects = nvme_command_effects(ctrl, ns, opcode); @@ -1753,7 +1932,8 @@ EXPORT_SYMBOL_GPL(nvme_set_queue_count); #define NVME_AEN_SUPPORTED \ (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \ - NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE) + NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_CCR_COMPLETE | \ + NVME_AEN_CFG_DISC_CHANGE) static void nvme_enable_aen(struct nvme_ctrl *ctrl) { @@ -3343,6 +3523,112 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, offset, 0); } +static enum rq_end_io_ret nvme_cancel_endio(struct request *req, + blk_status_t error, + const struct io_comp_batch *iob) +{ + struct nvme_ctrl *ctrl = req->end_io_data; + u32 result; + u16 imm_abrts, def_abrts; + + if (!error) { + result = le32_to_cpu(nvme_req(req)->result.u32); + def_abrts = upper_16_bits(result); + imm_abrts = lower_16_bits(result); + + dev_warn(ctrl->device, + "Cancel status: 0x0 imm abrts = %u def abrts = %u", + imm_abrts, def_abrts); + } else { + dev_warn(ctrl->device, "Cancel status: 0x%x", + nvme_req(req)->status); + } + + blk_mq_free_request(req); + return RQ_END_IO_NONE; +} + +int nvme_submit_cancel_req(struct nvme_ctrl *ctrl, struct request *rq, + unsigned int sqid, int action) +{ + struct nvme_command c = { }; + struct request *cancel_req; + + if (sqid == 0) + return -EINVAL; + + c.cancel.opcode = nvme_cmd_cancel; + c.cancel.sqid = cpu_to_le32(sqid); + c.cancel.nsid = NVME_NSID_ALL; + c.cancel.action = action; + if (action == NVME_CANCEL_ACTION_SINGLE_CMD) + c.cancel.cid = nvme_cid(rq); + else + c.cancel.cid = 0xFFFF; + + cancel_req = blk_mq_alloc_request_hctx(rq->q, nvme_req_op(&c), + BLK_MQ_REQ_NOWAIT | + BLK_MQ_REQ_RESERVED, + sqid - 1); + if (IS_ERR(cancel_req)) { + dev_warn(ctrl->device, "%s: Could not allocate the Cancel " + "command", __func__); + return PTR_ERR(cancel_req); + } + + nvme_init_request(cancel_req, &c); + cancel_req->end_io = nvme_cancel_endio; + cancel_req->end_io_data = ctrl; + + blk_execute_rq_nowait(cancel_req, false); + + return 0; +} +EXPORT_SYMBOL_GPL(nvme_submit_cancel_req); + +static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error, + const struct io_comp_batch *iob) +{ + struct nvme_ctrl *ctrl = req->end_io_data; + + dev_warn(ctrl->device, "Abort status: 0x%x", nvme_req(req)->status); + atomic_inc(&ctrl->abort_limit); + blk_mq_free_request(req); + return RQ_END_IO_NONE; +} + +int nvme_submit_abort_req(struct nvme_ctrl *ctrl, struct request *rq, + unsigned int sqid) +{ + struct nvme_command c = { }; + struct request *abort_req; + + c.abort.opcode = nvme_admin_abort_cmd; + c.abort.cid = nvme_cid(rq); + c.abort.sqid = sqid; + + if (atomic_dec_return(&ctrl->abort_limit) < 0) { + atomic_inc(&ctrl->abort_limit); + return -EBUSY; + } + + abort_req = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&c), + BLK_MQ_REQ_NOWAIT); + if (IS_ERR(abort_req)) { + atomic_inc(&ctrl->abort_limit); + return PTR_ERR(abort_req); + } + + nvme_init_request(abort_req, &c); + + abort_req->end_io = abort_endio; + abort_req->end_io_data = ctrl; + + blk_execute_rq_nowait(abort_req, false); + return 0; +} +EXPORT_SYMBOL_GPL(nvme_submit_abort_req); + static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi, struct nvme_effects_log **log) { @@ -3598,6 +3884,10 @@ static int nvme_init_identify(struct nvme_ctrl *ctrl) ctrl->crdt[1] = le16_to_cpu(id->crdt2); ctrl->crdt[2] = le16_to_cpu(id->crdt3); + ctrl->ciu = id->ciu; + ctrl->cirn = le64_to_cpu(id->cirn); + atomic_set(&ctrl->ccr_limit, id->ccrl); + ctrl->oacs = le16_to_cpu(id->oacs); ctrl->oncs = le16_to_cpu(id->oncs); ctrl->mtfa = le16_to_cpu(id->mtfa); @@ -4199,6 +4489,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) mutex_unlock(&ctrl->namespaces_lock); goto out_unlink_ns; } + blk_queue_rq_timeout(ns->queue, ctrl->io_timeout); nvme_ns_add_to_ctrl_list(ns); mutex_unlock(&ctrl->namespaces_lock); synchronize_srcu(&ctrl->srcu); @@ -4733,6 +5024,47 @@ static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl) kfree(log); } +static void nvme_ccr_work(struct work_struct *work) +{ + struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ccr_work); + struct nvme_ccr_entry *ccr; + struct nvme_ccr_log_entry *entry; + struct nvme_ccr_log *log; + unsigned long flags; + int ret, i; + + log = kmalloc(sizeof(*log), GFP_KERNEL); + if (!log) + return; + + ret = nvme_get_log(ctrl, 0, NVME_LOG_CCR, 0x01, + 0x00, log, sizeof(*log), 0); + if (ret) + goto out; + + spin_lock_irqsave(&ctrl->lock, flags); + for (i = 0; i < le16_to_cpu(log->ne); i++) { + entry = &log->entries[i]; + if (entry->ccrs == NVME_CCR_STATUS_IN_PROGRESS) + continue; + + list_for_each_entry(ccr, &ctrl->ccr_list, list) { + struct nvme_ctrl *ictrl = ccr->ictrl; + + if (ictrl->cntlid != le16_to_cpu(entry->icid) || + ictrl->ciu != entry->ciu) + continue; + + /* Complete matching entry */ + ccr->ccrs = entry->ccrs; + complete(&ccr->complete); + } + } + spin_unlock_irqrestore(&ctrl->lock, flags); +out: + kfree(log); +} + static void nvme_fw_act_work(struct work_struct *work) { struct nvme_ctrl *ctrl = container_of(work, @@ -4809,6 +5141,9 @@ static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) case NVME_AER_NOTICE_DISC_CHANGED: ctrl->aen_result = result; break; + case NVME_AER_NOTICE_CCR_COMPLETED: + queue_work(nvme_wq, &ctrl->ccr_work); + break; default: dev_warn(ctrl->device, "async event result %08x\n", result); } @@ -4885,12 +5220,7 @@ int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, if (ret) return ret; - /* - * If a previous admin queue exists (e.g., from before a reset), - * put it now before allocating a new one to avoid orphaning it. - */ - if (ctrl->admin_q) - blk_put_queue(ctrl->admin_q); + WARN_ON_ONCE(ctrl->admin_q); ctrl->admin_q = blk_mq_alloc_queue(set, &lim, NULL); if (IS_ERR(ctrl->admin_q)) { @@ -4928,10 +5258,8 @@ void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl) */ nvme_stop_keep_alive(ctrl); blk_mq_destroy_queue(ctrl->admin_q); - if (ctrl->ops->flags & NVME_F_FABRICS) { + if (ctrl->ops->flags & NVME_F_FABRICS) blk_mq_destroy_queue(ctrl->fabrics_q); - blk_put_queue(ctrl->fabrics_q); - } blk_mq_free_tag_set(ctrl->admin_tagset); } EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set); @@ -4951,9 +5279,14 @@ int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, */ if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS) set->reserved_tags = NVME_AQ_DEPTH; - else if (ctrl->ops->flags & NVME_F_FABRICS) + else if (ctrl->ops->flags & NVME_F_FABRICS) { /* Reserved for fabric connect */ set->reserved_tags = 1; + if (nvme_io_command_supported(ctrl, nvme_cmd_cancel)) { + /* Reserved for cancel commands */ + set->reserved_tags += NVME_RSV_CANCEL_MAX; + } + } set->numa_node = ctrl->numa_node; if (ctrl->ops->flags & NVME_F_BLOCKING) set->flags |= BLK_MQ_F_BLOCKING; @@ -5005,6 +5338,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl) nvme_stop_failfast_work(ctrl); flush_work(&ctrl->async_event_work); cancel_work_sync(&ctrl->fw_act_work); + cancel_work_sync(&ctrl->ccr_work); if (ctrl->ops->stop_ctrl) ctrl->ops->stop_ctrl(ctrl); } @@ -5073,6 +5407,8 @@ static void nvme_free_ctrl(struct device *dev) if (ctrl->admin_q) blk_put_queue(ctrl->admin_q); + if (ctrl->fabrics_q) + blk_put_queue(ctrl->fabrics_q); if (!subsys || ctrl->instance != subsys->instance) ida_free(&nvme_instance_ida, ctrl->instance); nvme_free_cels(ctrl); @@ -5121,12 +5457,14 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, mutex_init(&ctrl->scan_lock); INIT_LIST_HEAD(&ctrl->namespaces); + INIT_LIST_HEAD(&ctrl->ccr_list); xa_init(&ctrl->cels); ctrl->dev = dev; ctrl->ops = ops; ctrl->quirks = quirks; ctrl->numa_node = NUMA_NO_NODE; INIT_WORK(&ctrl->scan_work, nvme_scan_work); + INIT_WORK(&ctrl->ccr_work, nvme_ccr_work); INIT_WORK(&ctrl->async_event_work, nvme_async_event_work); INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work); INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work); @@ -5137,6 +5475,8 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd)); ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive; ctrl->ka_last_check_time = jiffies; + ctrl->admin_timeout = NVME_ADMIN_TIMEOUT; + ctrl->io_timeout = NVME_IO_TIMEOUT; BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) > PAGE_SIZE); @@ -5243,8 +5583,9 @@ void nvme_unfreeze(struct nvme_ctrl *ctrl) } EXPORT_SYMBOL_GPL(nvme_unfreeze); -int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout) +int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl) { + unsigned long timeout = ctrl->io_timeout; struct nvme_ns *ns; int srcu_idx; @@ -5252,7 +5593,7 @@ int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout) list_for_each_entry_srcu(ns, &ctrl->namespaces, list, srcu_read_lock_held(&ctrl->srcu)) { timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout); - if (timeout <= 0) + if (!timeout) break; } srcu_read_unlock(&ctrl->srcu, srcu_idx); @@ -5371,6 +5712,7 @@ static inline void _nvme_check_size(void) BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64); BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64); BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64); + BUILD_BUG_ON(sizeof(struct nvme_cancel_cmd) != 64); BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64); BUILD_BUG_ON(sizeof(struct nvme_command) != 64); BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index e4f4528fe2a2d6..1114a7d246b01f 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -166,12 +166,13 @@ struct nvme_fc_ctrl { struct blk_mq_tag_set admin_tag_set; struct blk_mq_tag_set tag_set; + struct work_struct fencing_work; struct work_struct ioerr_work; struct delayed_work connect_work; struct kref ref; unsigned long flags; - u32 iocnt; + atomic_t iocnt; wait_queue_head_t ioabort_wait; struct nvme_fc_fcp_op aen_ops[NVME_NR_AEN_COMMANDS]; @@ -227,6 +228,10 @@ static DEFINE_IDA(nvme_fc_ctrl_cnt); static struct device *fc_udev_device; static void nvme_fc_complete_rq(struct request *rq); +static void nvme_fc_start_ioerr_recovery(struct nvme_fc_ctrl *ctrl, + char *errmsg); +static void __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, + bool start_queues); /* *********************** FC-NVME Port Management ************************ */ @@ -788,7 +793,7 @@ nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl) "Reconnect", ctrl->cnum); set_bit(ASSOC_FAILED, &ctrl->flags); - nvme_reset_ctrl(&ctrl->ctrl); + nvme_fc_start_ioerr_recovery(ctrl, "Connectivity Loss"); } /** @@ -985,7 +990,7 @@ fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *); static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *); -static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg); +static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl); static void __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop) @@ -1567,9 +1572,8 @@ nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop) * for the association have been ABTS'd by * nvme_fc_delete_association(). */ - - /* fail the association */ - nvme_fc_error_recovery(ctrl, "Disconnect Association LS received"); + nvme_fc_start_ioerr_recovery(ctrl, + "Disconnect Association LS received"); /* release the reference taken by nvme_fc_match_disconn_ls() */ nvme_fc_ctrl_put(ctrl); @@ -1819,7 +1823,7 @@ __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op) atomic_set(&op->state, opstate); else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) { op->flags |= FCOP_FLAGS_TERMIO; - ctrl->iocnt++; + atomic_inc(&ctrl->iocnt); } spin_unlock_irqrestore(&ctrl->lock, flags); @@ -1849,20 +1853,45 @@ nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl) } static inline void +__nvme_fc_fcpop_count_one_down(struct nvme_fc_ctrl *ctrl) +{ + if (atomic_dec_return(&ctrl->iocnt) == 0) + wake_up(&ctrl->ioabort_wait); +} + +static inline bool __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op, int opstate) { unsigned long flags; + bool ret = false; if (opstate == FCPOP_STATE_ABORTED) { spin_lock_irqsave(&ctrl->lock, flags); if (test_bit(FCCTRL_TERMIO, &ctrl->flags) && op->flags & FCOP_FLAGS_TERMIO) { - if (!--ctrl->iocnt) - wake_up(&ctrl->ioabort_wait); + ret = true; } spin_unlock_irqrestore(&ctrl->lock, flags); } + + return ret; +} + +static void nvme_fc_fencing_work(struct work_struct *work) +{ + struct nvme_fc_ctrl *fc_ctrl = + container_of(work, struct nvme_fc_ctrl, fencing_work); + struct nvme_ctrl *ctrl = &fc_ctrl->ctrl; + int ret; + + ret = nvme_fence_ctrl(ctrl); + if (ret) + dev_info(ctrl->device, "CCR failed with error %d\n", ret); + + nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED); + if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) + queue_work(nvme_reset_wq, &fc_ctrl->ioerr_work); } static void @@ -1871,7 +1900,23 @@ nvme_fc_ctrl_ioerr_work(struct work_struct *work) struct nvme_fc_ctrl *ctrl = container_of(work, struct nvme_fc_ctrl, ioerr_work); - nvme_fc_error_recovery(ctrl, "transport detected io error"); + /* + * if an error (io timeout, etc) while (re)connecting, the remote + * port requested terminating of the association (disconnect_ls) + * or an error (timeout or abort) occurred on an io while creating + * the controller. Abort any ios on the association and let the + * create_association error path resolve things. + */ + if (nvme_ctrl_state(&ctrl->ctrl) == NVME_CTRL_CONNECTING) { + __nvme_fc_abort_outstanding_ios(ctrl, true); + dev_warn(ctrl->ctrl.device, + "NVME-FC{%d}: transport error during (re)connect\n", + ctrl->cnum); + return; + } + + flush_work(&ctrl->fencing_work); + nvme_fc_error_recovery(ctrl); } /* @@ -1892,6 +1937,32 @@ char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req) } EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid); +static void nvme_fc_start_ioerr_recovery(struct nvme_fc_ctrl *ctrl, + char *errmsg) +{ + enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl); + + if (state == NVME_CTRL_CONNECTING || state == NVME_CTRL_DELETING || + state == NVME_CTRL_DELETING_NOIO) { + queue_work(nvme_reset_wq, &ctrl->ioerr_work); + return; + } + + if (nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_FENCING)) { + dev_warn(ctrl->ctrl.device, + "NVME-FC{%d}: starting controller fencing %s\n", + ctrl->cnum, errmsg); + queue_work(nvme_wq, &ctrl->fencing_work); + return; + } + + if (nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING)) { + dev_warn(ctrl->ctrl.device, "NVME-FC{%d}: starting error recovery %s\n", + ctrl->cnum, errmsg); + queue_work(nvme_reset_wq, &ctrl->ioerr_work); + } +} + static void nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) { @@ -1904,7 +1975,8 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) struct nvme_command *sqe = &op->cmd_iu.sqe; __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1); union nvme_result result; - bool terminate_assoc = true; + bool op_term, terminate_assoc = true; + enum nvme_ctrl_state state; int opstate; /* @@ -2037,21 +2109,42 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) done: if (op->flags & FCOP_FLAGS_AEN) { nvme_complete_async_event(&queue->ctrl->ctrl, status, &result); - __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); + if (__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate)) + __nvme_fc_fcpop_count_one_down(ctrl); atomic_set(&op->state, FCPOP_STATE_IDLE); op->flags = FCOP_FLAGS_AEN; /* clear other flags */ nvme_fc_ctrl_put(ctrl); goto check_error; } - __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); + /* + * We can not access op after the request is completed because it can + * be reused immediately. At the same time we want to wakeup the thread + * waiting for ongoing IOs _after_ requests are completed. This is + * necessary because that thread will start canceling inflight IOs + * and we want to avoid request completion racing with cancellation. + */ + op_term = __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); + + /* + * If we are going to terminate associations and the controller is + * LIVE or FENCING, then do not complete this request now. Let error + * recovery cancel this request when it is safe to do so. + */ + state = nvme_ctrl_state(&ctrl->ctrl); + if (terminate_assoc && + (state == NVME_CTRL_LIVE || state == NVME_CTRL_FENCING)) + goto check_op_term; + if (!nvme_try_complete_req(rq, status, result)) nvme_fc_complete_rq(rq); +check_op_term: + if (op_term) + __nvme_fc_fcpop_count_one_down(ctrl); check_error: - if (terminate_assoc && - nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_RESETTING) - queue_work(nvme_reset_wq, &ctrl->ioerr_work); + if (terminate_assoc) + nvme_fc_start_ioerr_recovery(ctrl, "io error"); } static int @@ -2461,7 +2554,7 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues) * io requests back to the block layer as part of normal completions * (but with error status). */ - if (ctrl->ctrl.queue_count > 1) { + if (ctrl->ctrl.queue_count > 1 && ctrl->ctrl.tagset) { nvme_quiesce_io_queues(&ctrl->ctrl); nvme_sync_io_queues(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set, @@ -2495,39 +2588,6 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues) nvme_unquiesce_admin_queue(&ctrl->ctrl); } -static void -nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg) -{ - enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl); - - /* - * if an error (io timeout, etc) while (re)connecting, the remote - * port requested terminating of the association (disconnect_ls) - * or an error (timeout or abort) occurred on an io while creating - * the controller. Abort any ios on the association and let the - * create_association error path resolve things. - */ - if (state == NVME_CTRL_CONNECTING) { - __nvme_fc_abort_outstanding_ios(ctrl, true); - dev_warn(ctrl->ctrl.device, - "NVME-FC{%d}: transport error during (re)connect\n", - ctrl->cnum); - return; - } - - /* Otherwise, only proceed if in LIVE state - e.g. on first error */ - if (state != NVME_CTRL_LIVE) - return; - - dev_warn(ctrl->ctrl.device, - "NVME-FC{%d}: transport association event: %s\n", - ctrl->cnum, errmsg); - dev_warn(ctrl->ctrl.device, - "NVME-FC{%d}: resetting controller\n", ctrl->cnum); - - nvme_reset_ctrl(&ctrl->ctrl); -} - static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq) { struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); @@ -2536,24 +2596,14 @@ static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq) struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu; struct nvme_command *sqe = &cmdiu->sqe; - /* - * Attempt to abort the offending command. Command completion - * will detect the aborted io and will fail the connection. - */ dev_info(ctrl->ctrl.device, "NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: " "x%08x/x%08x\n", ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype, nvme_fabrics_opcode_str(qnum, sqe), sqe->common.cdw10, sqe->common.cdw11); - if (__nvme_fc_abort_op(ctrl, op)) - nvme_fc_error_recovery(ctrl, "io timeout abort failed"); - /* - * the io abort has been initiated. Have the reset timer - * restarted and the abort completion will complete the io - * shortly. Avoids a synchronous wait while the abort finishes. - */ + nvme_fc_start_ioerr_recovery(ctrl, "io timeout"); return BLK_EH_RESET_TIMER; } @@ -2729,7 +2779,8 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, * cmd with the csn was supposed to arrive. */ opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE); - __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); + if (__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate)) + __nvme_fc_fcpop_count_one_down(ctrl); if (!(op->flags & FCOP_FLAGS_AEN)) { nvme_fc_unmap_data(ctrl, op->rq, op); @@ -2900,6 +2951,11 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) out_delete_hw_queues: nvme_fc_delete_hw_io_queues(ctrl); out_cleanup_tagset: + /* + * In CONNECTING state ctrl->ioerr_work will abort both admin + * and io tagsets. Cancel it first before removing io tagset. + */ + cancel_work_sync(&ctrl->ioerr_work); nvme_remove_io_tag_set(&ctrl->ctrl); nvme_fc_free_io_queues(ctrl); @@ -3198,7 +3254,7 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) spin_lock_irqsave(&ctrl->lock, flags); set_bit(FCCTRL_TERMIO, &ctrl->flags); - ctrl->iocnt = 0; + atomic_set(&ctrl->iocnt, 0); spin_unlock_irqrestore(&ctrl->lock, flags); __nvme_fc_abort_outstanding_ios(ctrl, false); @@ -3207,11 +3263,19 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) nvme_fc_abort_aen_ops(ctrl); /* wait for all io that had to be aborted */ + wait_event(ctrl->ioabort_wait, atomic_read(&ctrl->iocnt) == 0); spin_lock_irq(&ctrl->lock); - wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock); clear_bit(FCCTRL_TERMIO, &ctrl->flags); spin_unlock_irq(&ctrl->lock); + /* + * At this point all inflight requests have been successfully + * aborted. Now it is safe to cancel all requests we decided + * not to complete in nvme_fc_fcpio_done(). + */ + nvme_cancel_tagset(&ctrl->ctrl); + nvme_cancel_admin_tagset(&ctrl->ctrl); + nvme_fc_term_aen_ops(ctrl); /* @@ -3329,6 +3393,7 @@ nvme_fc_reset_ctrl_work(struct work_struct *work) struct nvme_fc_ctrl *ctrl = container_of(work, struct nvme_fc_ctrl, ctrl.reset_work); + flush_work(&ctrl->fencing_work); nvme_stop_ctrl(&ctrl->ctrl); /* will block will waiting for io to terminate */ @@ -3352,6 +3417,27 @@ nvme_fc_reset_ctrl_work(struct work_struct *work) } } +static void +nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl) +{ + nvme_stop_keep_alive(&ctrl->ctrl); + nvme_stop_ctrl(&ctrl->ctrl); + flush_work(&ctrl->ctrl.async_event_work); + + /* will block while waiting for io to terminate */ + nvme_fc_delete_association(ctrl); + + /* Do not reconnect if controller is being deleted */ + if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) + return; + + if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) { + queue_delayed_work(nvme_wq, &ctrl->connect_work, 0); + return; + } + + nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN); +} static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = { .name = "fc", @@ -3483,6 +3569,7 @@ nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work); INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work); + INIT_WORK(&ctrl->fencing_work, nvme_fc_fencing_work); INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work); spin_lock_init(&ctrl->lock); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index ccd5e05dac98f0..163d562a449509 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -305,6 +305,8 @@ static inline u16 nvme_req_qid(struct request *req) enum nvme_ctrl_state { NVME_CTRL_NEW, NVME_CTRL_LIVE, + NVME_CTRL_FENCING, + NVME_CTRL_FENCED, NVME_CTRL_RESETTING, NVME_CTRL_CONNECTING, NVME_CTRL_DELETING, @@ -331,6 +333,13 @@ enum nvme_ctrl_flags { NVME_CTRL_FROZEN = 6, }; +struct nvme_ccr_entry { + struct list_head list; + struct completion complete; + struct nvme_ctrl *ictrl; + u8 ccrs; +}; + struct nvme_ctrl { bool comp_seen; bool identified; @@ -348,6 +357,7 @@ struct nvme_ctrl { struct blk_mq_tag_set *tagset; struct blk_mq_tag_set *admin_tagset; struct list_head namespaces; + struct list_head ccr_list; struct mutex namespaces_lock; struct srcu_struct srcu; struct device ctrl_device; @@ -370,6 +380,8 @@ struct nvme_ctrl { u16 mtfa; u32 ctrl_config; u32 queue_count; + u32 admin_timeout; + u32 io_timeout; u64 cap; u32 max_hw_sectors; @@ -382,11 +394,14 @@ struct nvme_ctrl { u16 crdt[3]; u16 oncs; u8 dmrl; + u8 ciu; u32 dmrsl; + u64 cirn; u16 oacs; u16 sqsize; u32 max_namespaces; atomic_t abort_limit; + atomic_t ccr_limit; u8 vwc; u32 vs; u32 sgls; @@ -406,6 +421,7 @@ struct nvme_ctrl { struct nvme_effects_log *effects; struct xarray cels; struct work_struct scan_work; + struct work_struct ccr_work; struct work_struct async_event_work; struct delayed_work ka_work; struct delayed_work failfast_work; @@ -834,6 +850,8 @@ static inline bool nvme_state_terminal(struct nvme_ctrl *ctrl) switch (nvme_ctrl_state(ctrl)) { case NVME_CTRL_NEW: case NVME_CTRL_LIVE: + case NVME_CTRL_FENCING: + case NVME_CTRL_FENCED: case NVME_CTRL_RESETTING: case NVME_CTRL_CONNECTING: return false; @@ -867,6 +885,7 @@ blk_status_t nvme_host_path_error(struct request *req); bool nvme_cancel_request(struct request *req, void *data); void nvme_cancel_tagset(struct nvme_ctrl *ctrl); void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl); +int nvme_fence_ctrl(struct nvme_ctrl *ctrl); bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, enum nvme_ctrl_state new_state); int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown); @@ -900,7 +919,7 @@ void nvme_sync_queues(struct nvme_ctrl *ctrl); void nvme_sync_io_queues(struct nvme_ctrl *ctrl); void nvme_unfreeze(struct nvme_ctrl *ctrl); void nvme_wait_freeze(struct nvme_ctrl *ctrl); -int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout); +int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl); void nvme_start_freeze(struct nvme_ctrl *ctrl); static inline enum req_op nvme_req_op(struct nvme_command *cmd) @@ -982,6 +1001,10 @@ int nvme_delete_ctrl(struct nvme_ctrl *ctrl); void nvme_queue_scan(struct nvme_ctrl *ctrl); int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, void *log, size_t size, u64 offset); +int nvme_submit_cancel_req(struct nvme_ctrl *ctrl, struct request *rq, + unsigned int sqid, int action); +int nvme_submit_abort_req(struct nvme_ctrl *ctrl, struct request *rq, + unsigned int sqid); bool nvme_tryget_ns_head(struct nvme_ns_head *head); void nvme_put_ns_head(struct nvme_ns_head *head); int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, @@ -1272,6 +1295,8 @@ static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {}; u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); +bool nvme_io_command_supported(struct nvme_ctrl *ctrl, u8 opcode); +bool nvme_is_cancel(struct nvme_command *cmd); u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); int nvme_execute_rq(struct request *rq, bool at_head); void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, @@ -1286,4 +1311,11 @@ static inline bool nvme_multi_css(struct nvme_ctrl *ctrl) return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI; } +static inline unsigned long nvme_fence_timeout_ms(struct nvme_ctrl *ctrl) +{ + if (ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) + return 3 * ctrl->kato * 1000; + return 2 * ctrl->kato * 1000; +} + #endif /* _NVME_H */ diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index c693308c6dd66e..3ed164b47a973e 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1776,18 +1776,6 @@ static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); } -static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error, - const struct io_comp_batch *iob) -{ - struct nvme_queue *nvmeq = req->mq_hctx->driver_data; - - dev_warn(nvmeq->dev->ctrl.device, - "Abort status: 0x%x", nvme_req(req)->status); - atomic_inc(&nvmeq->dev->ctrl.abort_limit); - blk_mq_free_request(req); - return RQ_END_IO_NONE; -} - static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) { /* If true, indicates loss of adapter communication, possibly by a @@ -1844,11 +1832,10 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) struct nvme_iod *iod = blk_mq_rq_to_pdu(req); struct nvme_queue *nvmeq = req->mq_hctx->driver_data; struct nvme_dev *dev = nvmeq->dev; - struct request *abort_req; - struct nvme_command cmd = { }; struct pci_dev *pdev = to_pci_dev(dev->dev); u32 csts = readl(dev->bar + NVME_REG_CSTS); u8 opcode; + int r; /* * Shutdown the device immediately if we see it is disconnected. This @@ -1935,11 +1922,6 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) atomic_inc(&dev->ctrl.abort_limit); return BLK_EH_RESET_TIMER; } - iod->flags |= IOD_ABORTED; - - cmd.abort.opcode = nvme_admin_abort_cmd; - cmd.abort.cid = nvme_cid(req); - cmd.abort.sqid = cpu_to_le16(nvmeq->qid); dev_warn(nvmeq->dev->ctrl.device, "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout, aborting req_op:%s(%u) size:%u\n", @@ -1947,17 +1929,11 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) nvmeq->qid, blk_op_str(req_op(req)), req_op(req), blk_rq_bytes(req)); - abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, nvme_req_op(&cmd), - BLK_MQ_REQ_NOWAIT); - if (IS_ERR(abort_req)) { - atomic_inc(&dev->ctrl.abort_limit); + r = nvme_submit_abort_req(&dev->ctrl, req, cpu_to_le16(nvmeq->qid)); + if (r == -EBUSY) return BLK_EH_RESET_TIMER; - } - nvme_init_request(abort_req, &cmd); - abort_req->end_io = abort_endio; - abort_req->end_io_data = NULL; - blk_execute_rq_nowait(abort_req, false); + iod->flags |= IOD_ABORTED; /* * The aborted req will be completed on receiving the abort req. @@ -3091,7 +3067,7 @@ static bool __nvme_delete_io_queues(struct nvme_dev *dev, u8 opcode) unsigned long timeout; retry: - timeout = NVME_ADMIN_TIMEOUT; + timeout = dev->ctrl.admin_timeout; while (nr_queues > 0) { if (nvme_delete_queue(&dev->queues[nr_queues], opcode)) break; @@ -3273,7 +3249,7 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) * if doing a safe shutdown. */ if (!dead && shutdown) - nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT); + nvme_wait_freeze_timeout(&dev->ctrl); } nvme_quiesce_io_queues(&dev->ctrl); diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index f77c960f7632d1..c0f4020ba5c4a0 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -74,12 +74,15 @@ struct nvme_rdma_request { struct nvme_rdma_sgl data_sgl; struct nvme_rdma_sgl *metadata_sgl; bool use_sig_mr; + bool aborted; }; enum nvme_rdma_queue_flags { NVME_RDMA_Q_ALLOCATED = 0, NVME_RDMA_Q_LIVE = 1, NVME_RDMA_Q_TR_READY = 2, + NVME_RDMA_Q_CANCEL_ONE = 3, + NVME_RDMA_Q_CANCEL_ALL = 4, }; struct nvme_rdma_queue { @@ -106,6 +109,7 @@ struct nvme_rdma_ctrl { /* other member variables */ struct blk_mq_tag_set tag_set; + struct work_struct fencing_work; struct work_struct err_work; struct nvme_rdma_qe async_event_sqe; @@ -619,6 +623,8 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl, } set_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags); + clear_bit(NVME_RDMA_Q_CANCEL_ONE, &queue->flags); + clear_bit(NVME_RDMA_Q_CANCEL_ALL, &queue->flags); return 0; @@ -888,7 +894,7 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) if (!new) { nvme_start_freeze(&ctrl->ctrl); nvme_unquiesce_io_queues(&ctrl->ctrl); - if (!nvme_wait_freeze_timeout(&ctrl->ctrl, NVME_IO_TIMEOUT)) { + if (!nvme_wait_freeze_timeout(&ctrl->ctrl)) { /* * If we timed out waiting for freeze we are likely to * be stuck. Fail the controller initialization just @@ -1120,11 +1126,28 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) nvme_rdma_reconnect_or_remove(ctrl, ret); } +static void nvme_rdma_fencing_work(struct work_struct *work) +{ + struct nvme_rdma_ctrl *rdma_ctrl = container_of(work, + struct nvme_rdma_ctrl, fencing_work); + struct nvme_ctrl *ctrl = &rdma_ctrl->ctrl; + int ret; + + ret = nvme_fence_ctrl(ctrl); + if (ret) + dev_info(ctrl->device, "CCR failed with error %d\n", ret); + + nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED); + if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) + queue_work(nvme_reset_wq, &rdma_ctrl->err_work); +} + static void nvme_rdma_error_recovery_work(struct work_struct *work) { struct nvme_rdma_ctrl *ctrl = container_of(work, struct nvme_rdma_ctrl, err_work); + flush_work(&ctrl->fencing_work); nvme_stop_keep_alive(&ctrl->ctrl); flush_work(&ctrl->ctrl.async_event_work); nvme_rdma_teardown_io_queues(ctrl, false); @@ -1147,6 +1170,12 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work) static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl) { + if (nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_FENCING)) { + dev_warn(ctrl->ctrl.device, "starting controller fencing\n"); + queue_work(nvme_wq, &ctrl->fencing_work); + return; + } + if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING)) return; @@ -1954,16 +1983,20 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq) { struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); struct nvme_rdma_queue *queue = req->queue; - struct nvme_rdma_ctrl *ctrl = queue->ctrl; + struct nvme_rdma_ctrl *rdma_ctrl = queue->ctrl; + struct nvme_ctrl *ctrl = &rdma_ctrl->ctrl; struct nvme_command *cmd = req->req.cmd; int qid = nvme_rdma_queue_idx(queue); + int error, action; + enum nvme_ctrl_state state; - dev_warn(ctrl->ctrl.device, + dev_warn(ctrl->device, "I/O tag %d (%04x) opcode %#x (%s) QID %d timeout\n", rq->tag, nvme_cid(rq), cmd->common.opcode, nvme_fabrics_opcode_str(qid, cmd), qid); - if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_LIVE) { + state = nvme_ctrl_state(ctrl); + if (state != NVME_CTRL_LIVE && state != NVME_CTRL_FENCING) { /* * If we are resetting, connecting or deleting we should * complete immediately because we may block controller @@ -1981,11 +2014,50 @@ static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq) return BLK_EH_DONE; } + if (!req->aborted) { + if (!qid) + goto err_recovery; + + if (!nvme_io_command_supported(ctrl, nvme_cmd_cancel)) { + error = nvme_submit_abort_req(ctrl, rq, qid); + if (error) { + if (error == -EBUSY) + return BLK_EH_RESET_TIMER; + goto err_recovery; + } + goto abort; + } + + if (!test_and_set_bit(NVME_RDMA_Q_CANCEL_ONE, &queue->flags)) { + action = NVME_CANCEL_ACTION_SINGLE_CMD; + } else if (!test_and_set_bit(NVME_RDMA_Q_CANCEL_ALL, + &queue->flags)) { + action = NVME_CANCEL_ACTION_MUL_CMD; + } else { + /* No free reserved commands. + * this means a "multiple commands" cancel + * is currently under execution and this request + * is likely to be canceled. Mark this + * request as aborted and reset the timer. + */ + goto abort; + } + + error = nvme_submit_cancel_req(ctrl, rq, qid, action); + if (error) + goto err_recovery; + +abort: + req->aborted = true; + return BLK_EH_RESET_TIMER; + } + /* * LIVE state should trigger the normal error recovery which will * handle completing this request. */ - nvme_rdma_error_recovery(ctrl); +err_recovery: + nvme_rdma_error_recovery(rdma_ctrl); return BLK_EH_RESET_TIMER; } @@ -2009,6 +2081,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); dev = queue->device->dev; + req->aborted = false; req->sqe.dma = ib_dma_map_single(dev, req->sqe.data, sizeof(struct nvme_command), @@ -2113,6 +2186,7 @@ static void nvme_rdma_complete_rq(struct request *rq) struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); struct nvme_rdma_queue *queue = req->queue; struct ib_device *ibdev = queue->device->dev; + bool is_cancel = nvme_is_cancel(req->req.cmd); if (req->use_sig_mr) nvme_rdma_check_pi_status(req); @@ -2121,6 +2195,10 @@ static void nvme_rdma_complete_rq(struct request *rq) ib_dma_unmap_single(ibdev, req->sqe.dma, sizeof(struct nvme_command), DMA_TO_DEVICE); nvme_complete_rq(rq); + if (is_cancel) { + if (!test_and_clear_bit(NVME_RDMA_Q_CANCEL_ALL, &queue->flags)) + clear_bit(NVME_RDMA_Q_CANCEL_ONE, &queue->flags); + } } static void nvme_rdma_map_queues(struct blk_mq_tag_set *set) @@ -2169,6 +2247,7 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work) container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work); int ret; + flush_work(&ctrl->fencing_work); nvme_stop_ctrl(&ctrl->ctrl); nvme_rdma_shutdown_ctrl(ctrl, false); @@ -2289,6 +2368,7 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev, INIT_DELAYED_WORK(&ctrl->reconnect_work, nvme_rdma_reconnect_ctrl_work); + INIT_WORK(&ctrl->fencing_work, nvme_rdma_fencing_work); INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work); INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work); diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index e59758616f277e..bd9014340cf4ae 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -388,6 +388,27 @@ nvme_show_int_function(queue_count); nvme_show_int_function(sqsize); nvme_show_int_function(kato); +static ssize_t nvme_sysfs_ciu_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%02x\n", ctrl->ciu); +} +static DEVICE_ATTR(ciu, S_IRUSR, nvme_sysfs_ciu_show, NULL); + +static ssize_t nvme_sysfs_cirn_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%016llx\n", ctrl->cirn); +} +static DEVICE_ATTR(cirn, S_IRUSR, nvme_sysfs_cirn_show, NULL); + + static ssize_t nvme_sysfs_delete(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -422,6 +443,8 @@ static ssize_t nvme_sysfs_show_state(struct device *dev, static const char *const state_name[] = { [NVME_CTRL_NEW] = "new", [NVME_CTRL_LIVE] = "live", + [NVME_CTRL_FENCING] = "fencing", + [NVME_CTRL_FENCED] = "fenced", [NVME_CTRL_RESETTING] = "resetting", [NVME_CTRL_CONNECTING] = "connecting", [NVME_CTRL_DELETING] = "deleting", @@ -623,6 +646,93 @@ static ssize_t quirks_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(quirks); +static ssize_t nvme_admin_timeout_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%u\n", + jiffies_to_msecs(ctrl->admin_timeout)); +} + +static ssize_t nvme_admin_timeout_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + u32 timeout; + int err; + + /* + * Wait until the controller reaches the LIVE state + * to be sure that admin_q and fabrics_q are + * properly initialized. + */ + if (!test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags)) + return -EBUSY; + + err = kstrtou32(buf, 10, &timeout); + if (err || !timeout) + return -EINVAL; + + ctrl->admin_timeout = msecs_to_jiffies(timeout); + + blk_queue_rq_timeout(ctrl->admin_q, ctrl->admin_timeout); + if (ctrl->fabrics_q) + blk_queue_rq_timeout(ctrl->fabrics_q, ctrl->admin_timeout); + + return count; +} + +static DEVICE_ATTR(admin_timeout, S_IRUGO | S_IWUSR, + nvme_admin_timeout_show, nvme_admin_timeout_store); + +static ssize_t nvme_io_timeout_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%u\n", jiffies_to_msecs(ctrl->io_timeout)); +} + +static ssize_t nvme_io_timeout_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + struct nvme_ns *ns; + u32 timeout; + int err; + + /* + * Wait until the controller reaches the LIVE state + * to be sure that connect_q is properly initialized. + */ + if (!test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags)) + return -EBUSY; + + err = kstrtou32(buf, 10, &timeout); + if (err || !timeout) + return -EINVAL; + + /* Take the namespaces_lock to avoid racing against nvme_alloc_ns() */ + mutex_lock(&ctrl->namespaces_lock); + + ctrl->io_timeout = msecs_to_jiffies(timeout); + list_for_each_entry(ns, &ctrl->namespaces, list) + blk_queue_rq_timeout(ns->queue, ctrl->io_timeout); + + if (ctrl->connect_q) + blk_queue_rq_timeout(ctrl->connect_q, ctrl->io_timeout); + + mutex_unlock(&ctrl->namespaces_lock); + + return count; +} + +static DEVICE_ATTR(io_timeout, S_IRUGO | S_IWUSR, + nvme_io_timeout_show, nvme_io_timeout_store); + #ifdef CONFIG_NVME_HOST_AUTH static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -756,6 +866,8 @@ static struct attribute *nvme_dev_attrs[] = { &dev_attr_numa_node.attr, &dev_attr_queue_count.attr, &dev_attr_sqsize.attr, + &dev_attr_ciu.attr, + &dev_attr_cirn.attr, &dev_attr_hostnqn.attr, &dev_attr_hostid.attr, &dev_attr_ctrl_loss_tmo.attr, @@ -765,6 +877,8 @@ static struct attribute *nvme_dev_attrs[] = { &dev_attr_cntrltype.attr, &dev_attr_dctype.attr, &dev_attr_quirks.attr, + &dev_attr_admin_timeout.attr, + &dev_attr_io_timeout.attr, #ifdef CONFIG_NVME_HOST_AUTH &dev_attr_dhchap_secret.attr, &dev_attr_dhchap_ctrl_secret.attr, diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 02c95c32b07e39..d31fdd09673348 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -123,6 +123,7 @@ struct nvme_tcp_request { size_t offset; size_t data_sent; enum nvme_tcp_send_state state; + bool aborted; }; enum nvme_tcp_queue_flags { @@ -130,6 +131,8 @@ enum nvme_tcp_queue_flags { NVME_TCP_Q_LIVE = 1, NVME_TCP_Q_POLLING = 2, NVME_TCP_Q_IO_CPU_SET = 3, + NVME_TCP_Q_CANCEL_ONE = 4, + NVME_TCP_Q_CANCEL_ALL = 5, }; enum nvme_tcp_recv_state { @@ -194,6 +197,7 @@ struct nvme_tcp_ctrl { struct sockaddr_storage src_addr; struct nvme_ctrl ctrl; + struct work_struct fencing_work; struct work_struct err_work; struct delayed_work connect_work; struct nvme_tcp_request async_req; @@ -206,6 +210,7 @@ static struct workqueue_struct *nvme_tcp_wq; static const struct blk_mq_ops nvme_tcp_mq_ops; static const struct blk_mq_ops nvme_tcp_admin_mq_ops; static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); +static void nvme_tcp_complete_rq(struct request *rq); static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) { @@ -612,6 +617,12 @@ static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) { + if (nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCING)) { + dev_warn(ctrl->device, "starting controller fencing\n"); + queue_work(nvme_wq, &to_tcp_ctrl(ctrl)->fencing_work); + return; + } + if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) return; @@ -639,7 +650,7 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, req->status = cqe->status; if (!nvme_try_complete_req(rq, req->status, cqe->result)) - nvme_complete_rq(rq); + nvme_tcp_complete_rq(rq); queue->nr_cqe++; return 0; @@ -905,7 +916,7 @@ static inline void nvme_tcp_end_request(struct request *rq, u16 status) union nvme_result res = {}; if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) - nvme_complete_rq(rq); + nvme_tcp_complete_rq(rq); } static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, @@ -1893,6 +1904,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, goto err_init_connect; set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); + clear_bit(NVME_TCP_Q_CANCEL_ONE, &queue->flags); + clear_bit(NVME_TCP_Q_CANCEL_ALL, &queue->flags); return 0; @@ -2194,7 +2207,7 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) if (!new) { nvme_start_freeze(ctrl); nvme_unquiesce_io_queues(ctrl); - if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { + if (!nvme_wait_freeze_timeout(ctrl)) { /* * If we timed out waiting for freeze we are likely to * be stuck. Fail the controller initialization just @@ -2471,12 +2484,29 @@ static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) nvme_tcp_reconnect_or_remove(ctrl, ret); } +static void nvme_tcp_fencing_work(struct work_struct *work) +{ + struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, + struct nvme_tcp_ctrl, fencing_work); + struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; + int ret; + + ret = nvme_fence_ctrl(ctrl); + if (ret) + dev_info(ctrl->device, "CCR failed with error %d\n", ret); + + nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED); + if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) + queue_work(nvme_reset_wq, &tcp_ctrl->err_work); +} + static void nvme_tcp_error_recovery_work(struct work_struct *work) { struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, struct nvme_tcp_ctrl, err_work); struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; + flush_work(&to_tcp_ctrl(ctrl)->fencing_work); if (nvme_tcp_key_revoke_needed(ctrl)) nvme_auth_revoke_tls_key(ctrl); nvme_stop_keep_alive(ctrl); @@ -2519,6 +2549,7 @@ static void nvme_reset_ctrl_work(struct work_struct *work) container_of(work, struct nvme_ctrl, reset_work); int ret; + flush_work(&to_tcp_ctrl(ctrl)->fencing_work); if (nvme_tcp_key_revoke_needed(ctrl)) nvme_auth_revoke_tls_key(ctrl); nvme_stop_ctrl(ctrl); @@ -2640,17 +2671,21 @@ static void nvme_tcp_complete_timed_out(struct request *rq) static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) { struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); - struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); struct nvme_command *cmd = &pdu->cmd; - int qid = nvme_tcp_queue_id(req->queue); + struct nvme_tcp_queue *queue = req->queue; + struct nvme_ctrl *ctrl = &queue->ctrl->ctrl; + int qid = nvme_tcp_queue_id(queue); + int error, action; + enum nvme_ctrl_state state; dev_warn(ctrl->device, "I/O tag %d (%04x) type %d opcode %#x (%s) QID %d timeout\n", rq->tag, nvme_cid(rq), pdu->hdr.type, cmd->common.opcode, nvme_fabrics_opcode_str(qid, cmd), qid); - if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) { + state = nvme_ctrl_state(ctrl); + if (state != NVME_CTRL_LIVE && state != NVME_CTRL_FENCING) { /* * If we are resetting, connecting or deleting we should * complete immediately because we may block controller @@ -2668,10 +2703,50 @@ static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) return BLK_EH_DONE; } + if (!req->aborted) { + if (!qid) + goto err_recovery; + + if (!nvme_io_command_supported(ctrl, nvme_cmd_cancel)) { + error = nvme_submit_abort_req(ctrl, rq, qid); + if (error) { + if (error == -EBUSY) + return BLK_EH_RESET_TIMER; + goto err_recovery; + } + + goto abort; + } + + if (!test_and_set_bit(NVME_TCP_Q_CANCEL_ONE, &queue->flags)) { + action = NVME_CANCEL_ACTION_SINGLE_CMD; + } else if (!test_and_set_bit(NVME_TCP_Q_CANCEL_ALL, + &queue->flags)) { + action = NVME_CANCEL_ACTION_MUL_CMD; + } else { + /* No free reserved commands. + * this means a "multiple commands" cancel + * is currently under execution and this request + * is likely to be canceled. Mark this + * request as aborted and reset the timer. + */ + goto abort; + } + + error = nvme_submit_cancel_req(ctrl, rq, qid, action); + if (error) + goto err_recovery; + +abort: + req->aborted = true; + return BLK_EH_RESET_TIMER; + } + /* * LIVE state should trigger the normal error recovery which will * handle completing this request. */ +err_recovery: nvme_tcp_error_recovery(ctrl); return BLK_EH_RESET_TIMER; } @@ -2716,6 +2791,7 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, req->pdu_len = 0; req->pdu_sent = 0; req->h2cdata_left = 0; + req->aborted = false; req->data_len = blk_rq_nr_phys_segments(rq) ? blk_rq_payload_bytes(rq) : 0; req->curr_bio = rq->bio; @@ -2832,10 +2908,24 @@ static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size) return len; } +static void nvme_tcp_complete_rq(struct request *rq) +{ + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + struct nvme_tcp_queue *queue = req->queue; + bool is_cancel = nvme_is_cancel(req->req.cmd); + + nvme_complete_rq(rq); + + if (is_cancel) { + if (!test_and_clear_bit(NVME_TCP_Q_CANCEL_ALL, &queue->flags)) + clear_bit(NVME_TCP_Q_CANCEL_ONE, &queue->flags); + } +} + static const struct blk_mq_ops nvme_tcp_mq_ops = { .queue_rq = nvme_tcp_queue_rq, .commit_rqs = nvme_tcp_commit_rqs, - .complete = nvme_complete_rq, + .complete = nvme_tcp_complete_rq, .init_request = nvme_tcp_init_request, .exit_request = nvme_tcp_exit_request, .init_hctx = nvme_tcp_init_hctx, @@ -2905,6 +2995,7 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, INIT_DELAYED_WORK(&ctrl->connect_work, nvme_tcp_reconnect_ctrl_work); + INIT_WORK(&ctrl->fencing_work, nvme_tcp_fencing_work); INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig index 4904097dfd490f..49bd56edcadce3 100644 --- a/drivers/nvme/target/Kconfig +++ b/drivers/nvme/target/Kconfig @@ -127,3 +127,12 @@ config NVME_TARGET_PCI_EPF capable PCI controller. If unsure, say N. + +config NVME_TARGET_DELAY_REQUESTS + bool "NVMe over Fabrics target request delay" + depends on NVME_TARGET && NVME_TARGET_DEBUGFS + help + This is a testing feature to allow delaying request completion in an + NVMe over Fabrics target, which allows for support of the cancel command. + + If unsure, say N. diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile index ed8522911d1f73..5d9848605285bc 100644 --- a/drivers/nvme/target/Makefile +++ b/drivers/nvme/target/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_NVME_TARGET_PCI_EPF) += nvmet-pci-epf.o nvmet-y += core.o configfs.o admin-cmd.o fabrics-cmd.o \ discovery.o io-cmd-file.o io-cmd-bdev.o pr.o +nvmet-$(CONFIG_NVME_TARGET_DELAY_REQUESTS) += io-cmd-cancel.o nvmet-$(CONFIG_NVME_TARGET_DEBUGFS) += debugfs.o nvmet-$(CONFIG_NVME_TARGET_PASSTHRU) += passthru.o nvmet-$(CONFIG_BLK_DEV_ZONED) += zns.o diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 3842087209da55..49e4c68472e7a7 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -220,6 +220,7 @@ static void nvmet_execute_get_supported_log_pages(struct nvmet_req *req) logs->lids[NVME_LOG_FEATURES] = cpu_to_le32(NVME_LIDS_LSUPP); logs->lids[NVME_LOG_RMI] = cpu_to_le32(NVME_LIDS_LSUPP); logs->lids[NVME_LOG_RESERVATION] = cpu_to_le32(NVME_LIDS_LSUPP); + logs->lids[NVME_LOG_CCR] = cpu_to_le32(NVME_LIDS_LSUPP); status = nvmet_copy_to_sgl(req, 0, logs, sizeof(*logs)); kfree(logs); @@ -376,7 +377,9 @@ static void nvmet_get_cmd_effects_admin(struct nvmet_ctrl *ctrl, log->acs[nvme_admin_get_features] = log->acs[nvme_admin_async_event] = log->acs[nvme_admin_keep_alive] = + log->acs[nvme_admin_cross_ctrl_reset] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP); + } static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log) @@ -389,6 +392,9 @@ static void nvmet_get_cmd_effects_nvm(struct nvme_effects_log *log) log->iocs[nvme_cmd_resv_release] = log->iocs[nvme_cmd_resv_report] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP); +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + log->iocs[nvme_cmd_cancel] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP); +#endif log->iocs[nvme_cmd_write] = log->iocs[nvme_cmd_write_zeroes] = cpu_to_le32(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC); @@ -605,6 +611,47 @@ static void nvmet_execute_get_log_page_features(struct nvmet_req *req) nvmet_req_complete(req, status); } +static void nvmet_execute_get_log_page_ccr(struct nvmet_req *req) +{ + struct nvmet_ctrl *ctrl = req->sq->ctrl; + struct nvmet_ccr *ccr; + struct nvme_ccr_log *log; + int index = 0; + u16 status; + + log = kzalloc(sizeof(*log), GFP_KERNEL); + if (!log) { + status = NVME_SC_INTERNAL; + goto out; + } + + mutex_lock(&ctrl->lock); + list_for_each_entry(ccr, &ctrl->ccr_list, entry) { + u8 flags = NVME_CCR_FLAGS_VALIDATED | NVME_CCR_FLAGS_INITIATED; + u8 status = ccr->ctrl ? NVME_CCR_STATUS_IN_PROGRESS : + NVME_CCR_STATUS_SUCCESS; + + log->entries[index].icid = cpu_to_le16(ccr->icid); + log->entries[index].ciu = ccr->ciu; + log->entries[index].acid = cpu_to_le16(0xffff); + log->entries[index].ccrs = status; + log->entries[index].ccrf = flags; + index++; + } + + /* Cleanup completed CCRs if requested */ + if (req->cmd->get_log_page.lsp & 0x1) + nvmet_ctrl_cleanup_ccrs(ctrl, false); + mutex_unlock(&ctrl->lock); + + log->ne = cpu_to_le16(index); + nvmet_clear_aen_bit(req, NVME_AEN_BIT_CCR_COMPLETE); + status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log)); + kfree(log); +out: + nvmet_req_complete(req, status); +} + static void nvmet_execute_get_log_page(struct nvmet_req *req) { if (!nvmet_check_transfer_len(req, nvmet_get_log_page_len(req->cmd))) @@ -638,6 +685,8 @@ static void nvmet_execute_get_log_page(struct nvmet_req *req) return nvmet_execute_get_log_page_rmi(req); case NVME_LOG_RESERVATION: return nvmet_execute_get_log_page_resv(req); + case NVME_LOG_CCR: + return nvmet_execute_get_log_page_ccr(req); } pr_debug("unhandled lid %d on qid %d\n", req->cmd->get_log_page.lid, req->sq->qid); @@ -691,6 +740,11 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) id->mdts = nvmet_ctrl_mdts(req); id->cntlid = cpu_to_le16(ctrl->cntlid); id->ver = cpu_to_le32(ctrl->subsys->ver); + if (!nvmet_is_disc_subsys(ctrl->subsys)) { + id->ciu = ctrl->ciu; + id->cirn = cpu_to_le64(ctrl->cirn); + id->ccrl = NVMF_CCR_LIMIT; + } /* XXX: figure out what to do about RTD3R/RTD3 */ id->oaes = cpu_to_le32(NVMET_AEN_CFG_OPTIONAL); @@ -1606,6 +1660,75 @@ void nvmet_execute_keep_alive(struct nvmet_req *req) nvmet_req_complete(req, status); } +void nvmet_execute_cross_ctrl_reset(struct nvmet_req *req) +{ + struct nvmet_ctrl *ictrl, *sctrl = req->sq->ctrl; + struct nvme_command *cmd = req->cmd; + struct nvmet_ccr *ccr, *new_ccr; + int ccr_active, ccr_total; + u16 cntlid, status = NVME_SC_SUCCESS; + + cntlid = le16_to_cpu(cmd->ccr.icid); + if (sctrl->cntlid == cntlid) { + req->error_loc = + offsetof(struct nvme_cross_ctrl_reset_cmd, icid); + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto out; + } + + /* Find and get impacted controller */ + ictrl = nvmet_ctrl_find_get_ccr(sctrl->subsys, sctrl->hostnqn, + cmd->ccr.ciu, cntlid, + le64_to_cpu(cmd->ccr.cirn)); + if (!ictrl) { + /* Immediate Reset Successful */ + nvmet_set_result(req, 1); + status = NVME_SC_SUCCESS; + goto out; + } + + ccr_total = ccr_active = 0; + mutex_lock(&sctrl->lock); + list_for_each_entry(ccr, &sctrl->ccr_list, entry) { + if (ccr->ctrl == ictrl) { + status = NVME_SC_CCR_IN_PROGRESS | NVME_STATUS_DNR; + goto out_unlock; + } + + ccr_total++; + if (ccr->ctrl) + ccr_active++; + } + + if (ccr_active >= NVMF_CCR_LIMIT) { + status = NVME_SC_CCR_LIMIT_EXCEEDED; + goto out_unlock; + } + if (ccr_total >= NVMF_CCR_PER_PAGE) { + status = NVME_SC_CCR_LOGPAGE_FULL; + goto out_unlock; + } + + new_ccr = kmalloc_obj(*new_ccr, GFP_KERNEL); + if (!new_ccr) { + status = NVME_SC_INTERNAL; + goto out_unlock; + } + + new_ccr->ciu = cmd->ccr.ciu; + new_ccr->icid = cntlid; + new_ccr->ctrl = ictrl; + list_add_tail(&new_ccr->entry, &sctrl->ccr_list); + +out_unlock: + mutex_unlock(&sctrl->lock); + if (status == NVME_SC_SUCCESS) + nvmet_ctrl_fatal_error(ictrl); + nvmet_ctrl_put(ictrl); +out: + nvmet_req_complete(req, status); +} + u32 nvmet_admin_cmd_data_len(struct nvmet_req *req) { struct nvme_command *cmd = req->cmd; @@ -1683,6 +1806,9 @@ u16 nvmet_parse_admin_cmd(struct nvmet_req *req) case nvme_admin_keep_alive: req->execute = nvmet_execute_keep_alive; return 0; + case nvme_admin_cross_ctrl_reset: + req->execute = nvmet_execute_cross_ctrl_reset; + return 0; default: return nvmet_report_invalid_opcode(req); } diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index a87567f40c9155..3facd83cdce1cf 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include #include @@ -115,6 +117,20 @@ u16 nvmet_zero_sgl(struct nvmet_req *req, off_t off, size_t len) return 0; } +void nvmet_ctrl_cleanup_ccrs(struct nvmet_ctrl *ctrl, bool all) +{ + struct nvmet_ccr *ccr, *tmp; + + lockdep_assert_held(&ctrl->lock); + + list_for_each_entry_safe(ccr, tmp, &ctrl->ccr_list, entry) { + if (all || ccr->ctrl == NULL) { + list_del(&ccr->entry); + kfree(ccr); + } + } +} + static u32 nvmet_max_nsid(struct nvmet_subsys *subsys) { struct nvmet_ns *cur; @@ -189,7 +205,7 @@ static void nvmet_async_event_work(struct work_struct *work) nvmet_async_events_process(ctrl); } -void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, +static void nvmet_add_async_event_locked(struct nvmet_ctrl *ctrl, u8 event_type, u8 event_info, u8 log_page) { struct nvmet_async_event *aen; @@ -202,13 +218,19 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, aen->event_info = event_info; aen->log_page = log_page; - mutex_lock(&ctrl->lock); list_add_tail(&aen->entry, &ctrl->async_events); - mutex_unlock(&ctrl->lock); queue_work(nvmet_wq, &ctrl->async_event_work); } +void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, + u8 event_info, u8 log_page) +{ + mutex_lock(&ctrl->lock); + nvmet_add_async_event_locked(ctrl, event_type, event_info, log_page); + mutex_unlock(&ctrl->lock); +} + static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid) { u32 i; @@ -803,10 +825,23 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status) nvmet_put_namespace(ns); } +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +static void nvmet_delayed_execute_req(struct work_struct *work); +#endif + void nvmet_req_complete(struct nvmet_req *req, u16 status) { struct nvmet_sq *sq = req->sq; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + unsigned long flags; + /* only need to update the xarray if this was a delayed request */ + if (req->req_work.work.func == nvmet_delayed_execute_req) { + xa_lock_irqsave(&sq->outstanding_requests, flags); + __xa_erase(&sq->outstanding_requests, req->cmd->common.command_id); + xa_unlock_irqrestore(&sq->outstanding_requests, flags); + } +#endif __nvmet_req_complete(req, status); percpu_ref_put(&sq->ref); } @@ -1027,6 +1062,9 @@ int nvmet_sq_init(struct nvmet_sq *sq, struct nvmet_cq *cq) nvmet_auth_sq_init(sq); sq->cq = cq; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + xa_init_flags(&sq->outstanding_requests, XA_FLAGS_LOCK_IRQ); +#endif return 0; } EXPORT_SYMBOL_GPL(nvmet_sq_init); @@ -1105,6 +1143,14 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req) if (nvmet_is_passthru_req(req)) return nvmet_parse_passthru_io_cmd(req); +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + if (req->cmd->common.opcode == nvme_cmd_cancel) { + req->execute = nvmet_execute_cancel; + if (req->cmd->cancel.nsid == NVME_NSID_ALL) + return 0; + } +#endif + ret = nvmet_req_find_ns(req); if (unlikely(ret)) return ret; @@ -1402,6 +1448,11 @@ static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl) return; } + if (!nvmet_is_disc_subsys(ctrl->subsys)) { + ctrl->ciu = ((u8)(ctrl->ciu + 1)) ? : 1; + ctrl->cirn = get_random_u64(); + nvmet_ctrl_cleanup_ccrs(ctrl, false); + } ctrl->csts = NVME_CSTS_RDY; /* @@ -1506,6 +1557,35 @@ struct nvmet_ctrl *nvmet_ctrl_find_get(const char *subsysnqn, return ctrl; } +struct nvmet_ctrl *nvmet_ctrl_find_get_ccr(struct nvmet_subsys *subsys, + const char *hostnqn, u8 ciu, + u16 cntlid, u64 cirn) +{ + struct nvmet_ctrl *ctrl, *ictrl = NULL; + bool found = false; + + mutex_lock(&subsys->lock); + list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { + if (ctrl->cntlid != cntlid) + continue; + + /* Avoid racing with a controller that is becoming ready */ + mutex_lock(&ctrl->lock); + if (ctrl->ciu == ciu && ctrl->cirn == cirn) + found = true; + mutex_unlock(&ctrl->lock); + + if (found) { + if (kref_get_unless_zero(&ctrl->ref)) + ictrl = ctrl; + break; + } + }; + mutex_unlock(&subsys->lock); + + return ictrl; +} + u16 nvmet_check_ctrl_status(struct nvmet_req *req) { if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) { @@ -1631,6 +1711,7 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) subsys->clear_ids = 1; #endif + INIT_LIST_HEAD(&ctrl->ccr_list); INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work); INIT_LIST_HEAD(&ctrl->async_events); INIT_RADIX_TREE(&ctrl->p2p_ns_map, GFP_KERNEL); @@ -1667,6 +1748,11 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) } ctrl->cntlid = ret; + if (!nvmet_is_disc_subsys(ctrl->subsys)) { + ctrl->ciu = get_random_u8() ? : 1; + ctrl->cirn = get_random_u64(); + } + /* * Discovery controllers may use some arbitrary high value * in order to cleanup stale discovery sessions @@ -1736,12 +1822,56 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) } EXPORT_SYMBOL_GPL(nvmet_alloc_ctrl); +static void nvmet_ctrl_notify_ccr(struct nvmet_ctrl *ctrl) +{ + lockdep_assert_held(&ctrl->lock); + + if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_CCR_COMPLETE)) + return; + + nvmet_add_async_event_locked(ctrl, NVME_AER_NOTICE, + NVME_AER_NOTICE_CCR_COMPLETED, + NVME_LOG_CCR); +} + +static void nvmet_ctrl_complete_pending_ccr(struct nvmet_ctrl *ctrl) +{ + struct nvmet_subsys *subsys = ctrl->subsys; + struct nvmet_ctrl *sctrl; + struct nvmet_ccr *ccr; + + lockdep_assert_held(&subsys->lock); + + /* Cleanup all CCRs issued by ctrl as source controller */ + mutex_lock(&ctrl->lock); + nvmet_ctrl_cleanup_ccrs(ctrl, true); + mutex_unlock(&ctrl->lock); + + /* + * Find all CCRs targeting ctrl as impacted controller and + * set ccr->ctrl to NULL. This tells the source controller + * that CCR completed successfully. + */ + list_for_each_entry(sctrl, &subsys->ctrls, subsys_entry) { + mutex_lock(&sctrl->lock); + list_for_each_entry(ccr, &sctrl->ccr_list, entry) { + if (ccr->ctrl == ctrl) { + ccr->ctrl = NULL; + nvmet_ctrl_notify_ccr(sctrl); + break; + } + } + mutex_unlock(&sctrl->lock); + } +} + static void nvmet_ctrl_free(struct kref *ref) { struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref); struct nvmet_subsys *subsys = ctrl->subsys; mutex_lock(&subsys->lock); + nvmet_ctrl_complete_pending_ccr(ctrl); nvmet_ctrl_destroy_pr(ctrl); nvmet_release_p2p_ns_map(ctrl); list_del(&ctrl->subsys_entry); @@ -1792,6 +1922,46 @@ ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl, return ctrl->ops->host_traddr(ctrl, traddr, traddr_len); } +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +static void nvmet_delayed_execute_req(struct work_struct *work) { + struct nvmet_req *req = + container_of(to_delayed_work(work), struct nvmet_req, req_work); + req->execute(req); +} + +void nvmet_execute_request(struct nvmet_req *req) { + struct nvmet_ctrl *ctrl = req->sq->ctrl; + int delay_count; + u32 delay_msec; + unsigned long flags; + + if (unlikely(req->sq->qid == 0)) + return req->execute(req); + + if (ctrl) { + delay_count = atomic_dec_if_positive(&ctrl->delay_count) + 1; + delay_msec = ctrl->delay_msec; + } + if (!(ctrl && delay_count && delay_msec)) + return req->execute(req); + + xa_lock_irqsave(&req->sq->outstanding_requests, flags); + int ret = __xa_insert(&req->sq->outstanding_requests, + req->cmd->common.command_id, req, GFP_KERNEL); + xa_unlock_irqrestore(&req->sq->outstanding_requests, flags); + + if (ret) { + pr_err("nvmet: failure to delay command %d", + req->cmd->common.command_id); + return req->execute(req); + } + + INIT_DELAYED_WORK(&req->req_work, nvmet_delayed_execute_req); + queue_delayed_work(nvmet_wq, &req->req_work, msecs_to_jiffies(delay_msec)); +} +EXPORT_SYMBOL_GPL(nvmet_execute_request); +#endif + static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, const char *subsysnqn) { diff --git a/drivers/nvme/target/debugfs.c b/drivers/nvme/target/debugfs.c index 5dcbd5aa86e194..8315b215a5827b 100644 --- a/drivers/nvme/target/debugfs.c +++ b/drivers/nvme/target/debugfs.c @@ -152,6 +152,59 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p) } NVMET_DEBUGFS_ATTR(nvmet_ctrl_tls_concat); #endif +static int nvmet_ctrl_instance_ciu_show(struct seq_file *m, void *p) +{ + struct nvmet_ctrl *ctrl = m->private; + + seq_printf(m, "%02x\n", ctrl->ciu); + return 0; +} +NVMET_DEBUGFS_ATTR(nvmet_ctrl_instance_ciu); + +static int nvmet_ctrl_instance_cirn_show(struct seq_file *m, void *p) +{ + struct nvmet_ctrl *ctrl = m->private; + + seq_printf(m, "%016llx\n", ctrl->cirn); + return 0; +} +NVMET_DEBUGFS_ATTR(nvmet_ctrl_instance_cirn); + +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +static int nvmet_ctrl_delay_show(struct seq_file *m, void *p) +{ + struct nvmet_ctrl *ctrl = m->private; + int delay_count = atomic_read(&ctrl->delay_count); + + seq_printf(m, "%u %u\n", delay_count, ctrl->delay_msec); + return 0; +} + +static ssize_t nvmet_ctrl_delay_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = file->private_data; + struct nvmet_ctrl *ctrl = m->private; + char delay_buf[22] = {}; + int delay_count; + int delay_msec; + int n; + + if (count >= sizeof(delay_buf)) + return -EINVAL; + if (copy_from_user(delay_buf, buf, count)) + return -EFAULT; + + n = sscanf(delay_buf, "%u %u", &delay_count, &delay_msec); + if (n < 1 || n > 2) + return -EINVAL; + if (n == 2) + ctrl->delay_msec = delay_msec; + atomic_set(&ctrl->delay_count, delay_count); + return count; +} +NVMET_DEBUGFS_RW_ATTR(nvmet_ctrl_delay); +#endif /* CONFIG_NVME_TARGET_DELAY_REQUESTS */ int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) { @@ -184,6 +237,14 @@ int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) debugfs_create_file("tls_key", S_IRUSR, ctrl->debugfs_dir, ctrl, &nvmet_ctrl_tls_key_fops); #endif +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + debugfs_create_file("delay", S_IWUSR, ctrl->debugfs_dir, ctrl, + &nvmet_ctrl_delay_fops); +#endif + debugfs_create_file("ciu", S_IRUSR, ctrl->debugfs_dir, ctrl, + &nvmet_ctrl_instance_ciu_fops); + debugfs_create_file("cirn", S_IRUSR, ctrl->debugfs_dir, ctrl, + &nvmet_ctrl_instance_cirn_fops); return 0; } diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index d161707559ce69..36cc57005969d8 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -2390,7 +2390,7 @@ nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod) } /* data transfer complete, resume with nvmet layer */ - fod->req.execute(&fod->req); + nvmet_execute_request(&fod->req); break; case NVMET_FCOP_READDATA: @@ -2599,7 +2599,7 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, * can invoke the nvmet_layer now. If read data, cmd completion will * push the data */ - fod->req.execute(&fod->req); + nvmet_execute_request(&fod->req); return; transport_error: diff --git a/drivers/nvme/target/io-cmd-cancel.c b/drivers/nvme/target/io-cmd-cancel.c new file mode 100644 index 00000000000000..8d7f363c916376 --- /dev/null +++ b/drivers/nvme/target/io-cmd-cancel.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NVMe I/O cancel command implementation. + * Copyright (c) 2023 Red Hat + */ + +#include "nvmet.h" + +void nvmet_execute_cancel(struct nvmet_req *req) +{ + u16 cid; + __le16 sqid; + bool mult_cmds; + int ret = 0; + struct nvmet_ctrl *ctrl = req->sq->ctrl; + struct nvmet_sq *sq = req->sq; + struct nvmet_req *treq; + u32 canceled = 0; + + if (!nvmet_check_transfer_len(req, 0)) + return; + + sqid = le16_to_cpu(req->cmd->cancel.sqid); + if (sqid > ctrl->subsys->max_qid) { + ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto exit; + } + + mult_cmds = req->cmd->cancel.action & NVME_CANCEL_ACTION_MUL_CMD; + cid = req->cmd->cancel.cid; + + if (cid == req->cmd->cancel.command_id && !mult_cmds) { + /* If action is set to "single command" and cid is + * set to the cid of this cancel command, then + * the controller shall abort the command with + * an "invalid cid" status code. + */ + ret = NVME_SC_INVALID_CID | NVME_STATUS_DNR; + } else if ((cid != 0xFFFF && mult_cmds) || sqid != sq->qid) { + /* if action is set to "multiple commands" and + * cid isn't set to 0xFFFF, then abort the command + * with an "invalid field" status. + * if the sqid field doesn't match the sqid of + * the queue to which the cancel command is submitted, + * then abort the command with an "invalid field" status. + */ + ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + } + + if (!mult_cmds) { + treq = xa_load(&sq->outstanding_requests, cid); + if (treq) { + if (cancel_delayed_work(&treq->req_work)) { + pr_info("nvmet: CANCEL success: %d", cid); + nvmet_req_complete(treq, NVME_SC_ABORT_REQ); + canceled += 1; + } else { + pr_info("nvmet: CANCEL failed: %d", cid); + } + } else { + pr_info("nvmet: CANCEL request not found: %d", cid); + } + } else { + unsigned long ucid; + xa_for_each(&sq->outstanding_requests, ucid, treq) { + if (cancel_delayed_work(&treq->req_work)) { + nvmet_req_complete(treq, NVME_SC_ABORT_REQ); + canceled += 1; + } + } + pr_info("nvmet: CANCEL removed %d requests", canceled); + } +exit: + nvmet_set_result(req, canceled); + nvmet_req_complete(req, ret); +} + diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index d98d0cdc5d6fad..e521d5810150d0 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -127,7 +127,7 @@ static void nvme_loop_execute_work(struct work_struct *work) struct nvme_loop_iod *iod = container_of(work, struct nvme_loop_iod, work); - iod->req.execute(&iod->req); + nvmet_execute_request(&iod->req); } static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, @@ -261,7 +261,7 @@ static const struct blk_mq_ops nvme_loop_admin_mq_ops = { .init_hctx = nvme_loop_init_admin_hctx, }; -static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) +static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl, bool remove) { if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags)) return; @@ -274,7 +274,8 @@ static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); nvmet_cq_put(&ctrl->queues[0].nvme_cq); - nvme_remove_admin_tag_set(&ctrl->ctrl); + if (remove) + nvme_remove_admin_tag_set(&ctrl->ctrl); } static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) @@ -361,7 +362,7 @@ static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl) return 0; } -static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) +static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl, bool new) { int error; @@ -375,12 +376,15 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) } ctrl->ctrl.queue_count = 1; - error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set, - &nvme_loop_admin_mq_ops, - sizeof(struct nvme_loop_iod) + - NVME_INLINE_SG_CNT * sizeof(struct scatterlist)); - if (error) - goto out_free_sq; + if (new) { + error = nvme_alloc_admin_tag_set(&ctrl->ctrl, + &ctrl->admin_tag_set, + &nvme_loop_admin_mq_ops, + sizeof(struct nvme_loop_iod) + + NVME_INLINE_SG_CNT * sizeof(struct scatterlist)); + if (error) + goto out_free_sq; + } /* reset stopped state for the fresh admin queue */ clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags); @@ -415,7 +419,7 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) return error; } -static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl) +static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl, bool remove) { if (ctrl->ctrl.queue_count > 1) { nvme_quiesce_io_queues(&ctrl->ctrl); @@ -426,12 +430,12 @@ static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl) if (nvme_ctrl_state(&ctrl->ctrl) == NVME_CTRL_LIVE) nvme_disable_ctrl(&ctrl->ctrl, true); - nvme_loop_destroy_admin_queue(ctrl); + nvme_loop_destroy_admin_queue(ctrl, remove); } static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl) { - nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl)); + nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl), true); } static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl) @@ -453,7 +457,7 @@ static void nvme_loop_reset_ctrl_work(struct work_struct *work) int ret; nvme_stop_ctrl(&ctrl->ctrl); - nvme_loop_shutdown_ctrl(ctrl); + nvme_loop_shutdown_ctrl(ctrl, false); if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl); @@ -465,7 +469,7 @@ static void nvme_loop_reset_ctrl_work(struct work_struct *work) return; } - ret = nvme_loop_configure_admin_queue(ctrl); + ret = nvme_loop_configure_admin_queue(ctrl, false); if (ret) goto out_disable; @@ -492,7 +496,7 @@ static void nvme_loop_reset_ctrl_work(struct work_struct *work) out_destroy_admin: nvme_quiesce_admin_queue(&ctrl->ctrl); nvme_cancel_admin_tagset(&ctrl->ctrl); - nvme_loop_destroy_admin_queue(ctrl); + nvme_loop_destroy_admin_queue(ctrl, true); out_disable: dev_warn(ctrl->ctrl.device, "Removing after reset failure\n"); nvme_uninit_ctrl(&ctrl->ctrl); @@ -594,7 +598,7 @@ static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev, if (!ctrl->queues) goto out_uninit_ctrl; - ret = nvme_loop_configure_admin_queue(ctrl); + ret = nvme_loop_configure_admin_queue(ctrl, true); if (ret) goto out_free_queues; @@ -632,7 +636,7 @@ static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev, out_remove_admin_queue: nvme_quiesce_admin_queue(&ctrl->ctrl); nvme_cancel_admin_tagset(&ctrl->ctrl); - nvme_loop_destroy_admin_queue(ctrl); + nvme_loop_destroy_admin_queue(ctrl, true); out_free_queues: kfree(ctrl->queues); out_uninit_ctrl: diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 64af6bf569bf45..2b01cef1fb8984 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -44,7 +44,8 @@ * Supported optional AENs: */ #define NVMET_AEN_CFG_OPTIONAL \ - (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_ANA_CHANGE) + (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_ANA_CHANGE | \ + NVME_AEN_CFG_CCR_COMPLETE) #define NVMET_DISC_AEN_CFG_OPTIONAL \ (NVME_AEN_CFG_DISC_CHANGE) @@ -174,6 +175,9 @@ struct nvmet_sq { #endif struct completion free_done; struct completion confirm_done; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + struct xarray outstanding_requests; +#endif }; struct nvmet_ana_group { @@ -265,8 +269,11 @@ struct nvmet_ctrl { uuid_t hostid; u16 cntlid; + u8 ciu; u32 kato; + u64 cirn; + struct list_head ccr_list; struct nvmet_port *port; u32 aen_enabled; @@ -309,10 +316,21 @@ struct nvmet_ctrl { #endif #ifdef CONFIG_NVME_TARGET_TCP_TLS struct key *tls_key; +#endif +#ifdef CONFIG_NVME_TARGET_DELAY_REQUESTS + atomic_t delay_count; + u32 delay_msec; #endif struct nvmet_pr_log_mgr pr_log_mgr; }; +struct nvmet_ccr { + struct nvmet_ctrl *ctrl; + struct list_head entry; + u16 icid; + u8 ciu; +}; + struct nvmet_subsys { enum nvme_subsys_type type; @@ -495,6 +513,9 @@ struct nvmet_req { u16 error_loc; u64 error_slba; struct nvmet_pr_per_ctrl_ref *pc_ref; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) + struct delayed_work req_work; +#endif }; #define NVMET_MAX_MPOOL_BVEC 16 @@ -571,12 +592,14 @@ size_t nvmet_req_transfer_len(struct nvmet_req *req); bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len); bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len); void nvmet_req_complete(struct nvmet_req *req, u16 status); +void nvmet_req_complete_delayed(struct nvmet_req *req, u16 status); int nvmet_req_alloc_sgls(struct nvmet_req *req); void nvmet_req_free_sgls(struct nvmet_req *req); void nvmet_execute_set_features(struct nvmet_req *req); void nvmet_execute_get_features(struct nvmet_req *req); void nvmet_execute_keep_alive(struct nvmet_req *req); +void nvmet_execute_cross_ctrl_reset(struct nvmet_req *req); u16 nvmet_check_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create); u16 nvmet_check_io_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create); @@ -619,6 +642,10 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args); struct nvmet_ctrl *nvmet_ctrl_find_get(const char *subsysnqn, const char *hostnqn, u16 cntlid, struct nvmet_req *req); +struct nvmet_ctrl *nvmet_ctrl_find_get_ccr(struct nvmet_subsys *subsys, + const char *hostnqn, u8 ciu, + u16 cntlid, u64 cirn); +void nvmet_ctrl_cleanup_ccrs(struct nvmet_ctrl *ctrl, bool all); void nvmet_ctrl_put(struct nvmet_ctrl *ctrl); u16 nvmet_check_ctrl_status(struct nvmet_req *req); ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl, @@ -727,6 +754,8 @@ void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req); void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req); void nvmet_bdev_execute_zone_append(struct nvmet_req *req); +void nvmet_execute_cancel(struct nvmet_req *req); + static inline u32 nvmet_rw_data_len(struct nvmet_req *req) { return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) << @@ -989,4 +1018,10 @@ struct nvmet_feat_arbitration { u8 ab; }; +#if IS_ENABLED(CONFIG_NVME_TARGET_DELAY_REQUESTS) +void nvmet_execute_request(struct nvmet_req *req); +#else +static inline void nvmet_execute_request(struct nvmet_req *req) { req->execute(req); } +#endif + #endif /* _NVMET_H */ diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 2d6eb89f98af2d..c9cb70e30788b2 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -772,7 +772,7 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc) if (unlikely(status)) nvmet_req_complete(&rsp->req, status); else - rsp->req.execute(&rsp->req); + nvmet_execute_request(&rsp->req); } static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc) @@ -957,7 +957,7 @@ static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp) queue->cm_id->port_num, &rsp->read_cqe, NULL)) nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR); } else { - rsp->req.execute(&rsp->req); + nvmet_execute_request(&rsp->req); } return true; diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 164a564ba3b4e9..ef7cdc96ad57e0 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -622,7 +622,7 @@ static void nvmet_tcp_execute_request(struct nvmet_tcp_cmd *cmd) if (unlikely(cmd->flags & NVMET_TCP_F_INIT_FAILED)) nvmet_tcp_queue_response(&cmd->req); else - cmd->req.execute(&cmd->req); + nvmet_execute_request(&cmd->req); } static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd) @@ -1116,7 +1116,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) goto out; } - queue->cmd->req.execute(&queue->cmd->req); + nvmet_execute_request(&queue->cmd->req); out: nvmet_prepare_receive_pdu(queue); return ret; diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 041f30931a908a..049e8bf538f485 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -21,6 +21,9 @@ #define NVMF_TRADDR_SIZE 256 #define NVMF_TSAS_SIZE 256 +#define NVMF_CCR_LIMIT 4 +#define NVMF_CCR_PER_PAGE 511 + #define NVME_DISC_SUBSYS_NAME "nqn.2014-08.org.nvmexpress.discovery" #define NVME_NSID_ALL 0xffffffff @@ -28,6 +31,9 @@ /* Special NSSR value, 'NVMe' */ #define NVME_SUBSYS_RESET 0x4E564D65 +/* Maximum number of reserved commands for Cancel */ +#define NVME_RSV_CANCEL_MAX 2 + enum nvme_subsys_type { /* Referral to another discovery type target subsystem */ NVME_NQN_DISC = 1, @@ -328,7 +334,10 @@ struct nvme_id_ctrl { __le16 crdt1; __le16 crdt2; __le16 crdt3; - __u8 rsvd134[122]; + __u8 rsvd134[1]; + __u8 ciu; + __le64 cirn; + __u8 rsvd144[112]; __le16 oacs; __u8 acl; __u8 aerl; @@ -389,7 +398,8 @@ struct nvme_id_ctrl { __u8 msdbd; __u8 rsvd1804[2]; __u8 dctype; - __u8 rsvd1807[241]; + __u8 ccrl; + __u8 rsvd1808[240]; struct nvme_id_power_state psd[32]; __u8 vs[1024]; }; @@ -864,12 +874,14 @@ enum { NVME_AER_NOTICE_FW_ACT_STARTING = 0x01, NVME_AER_NOTICE_ANA = 0x03, NVME_AER_NOTICE_DISC_CHANGED = 0xf0, + NVME_AER_NOTICE_CCR_COMPLETED = 0xf4, }; enum { NVME_AEN_BIT_NS_ATTR = 8, NVME_AEN_BIT_FW_ACT = 9, NVME_AEN_BIT_ANA_CHANGE = 11, + NVME_AEN_BIT_CCR_COMPLETE = 20, NVME_AEN_BIT_DISC_CHANGE = 31, }; @@ -877,6 +889,7 @@ enum { NVME_AEN_CFG_NS_ATTR = 1 << NVME_AEN_BIT_NS_ATTR, NVME_AEN_CFG_FW_ACT = 1 << NVME_AEN_BIT_FW_ACT, NVME_AEN_CFG_ANA_CHANGE = 1 << NVME_AEN_BIT_ANA_CHANGE, + NVME_AEN_CFG_CCR_COMPLETE = 1 << NVME_AEN_BIT_CCR_COMPLETE, NVME_AEN_CFG_DISC_CHANGE = 1 << NVME_AEN_BIT_DISC_CHANGE, }; @@ -967,6 +980,7 @@ enum nvme_opcode { nvme_cmd_resv_acquire = 0x11, nvme_cmd_io_mgmt_recv = 0x12, nvme_cmd_resv_release = 0x15, + nvme_cmd_cancel = 0x18, nvme_cmd_zone_mgmt_send = 0x79, nvme_cmd_zone_mgmt_recv = 0x7a, nvme_cmd_zone_append = 0x7d, @@ -1227,6 +1241,22 @@ struct nvme_zone_mgmt_recv_cmd { __le32 cdw14[2]; }; +struct nvme_cross_ctrl_reset_cmd { + __u8 opcode; + __u8 flags; + __u16 command_id; + __le32 nsid; + __le64 rsvd2[2]; + union nvme_data_ptr dptr; + __le16 icid; + __u8 ciu; + __u8 rsvd10; + __le32 cdw11; + __le64 cirn; + __le32 cdw14; + __le32 cdw15; +}; + struct nvme_io_mgmt_recv_cmd { __u8 opcode; __u8 flags; @@ -1325,6 +1355,7 @@ enum nvme_admin_opcode { nvme_admin_virtual_mgmt = 0x1c, nvme_admin_nvme_mi_send = 0x1d, nvme_admin_nvme_mi_recv = 0x1e, + nvme_admin_cross_ctrl_reset = 0x38, nvme_admin_dbbuf = 0x7C, nvme_admin_format_nvm = 0x80, nvme_admin_security_send = 0x81, @@ -1358,6 +1389,7 @@ enum nvme_admin_opcode { nvme_admin_opcode_name(nvme_admin_virtual_mgmt), \ nvme_admin_opcode_name(nvme_admin_nvme_mi_send), \ nvme_admin_opcode_name(nvme_admin_nvme_mi_recv), \ + nvme_admin_opcode_name(nvme_admin_cross_ctrl_reset), \ nvme_admin_opcode_name(nvme_admin_dbbuf), \ nvme_admin_opcode_name(nvme_admin_format_nvm), \ nvme_admin_opcode_name(nvme_admin_security_send), \ @@ -1418,6 +1450,7 @@ enum { NVME_LOG_FDP_CONFIGS = 0x20, NVME_LOG_DISC = 0x70, NVME_LOG_RESERVATION = 0x80, + NVME_LOG_CCR = 0x1E, NVME_FWACT_REPL = (0 << 3), NVME_FWACT_REPL_ACTV = (1 << 3), NVME_FWACT_ACTV = (2 << 3), @@ -1441,6 +1474,34 @@ enum { NVME_FIS_CSCPE = 1 << 21, }; +/* NVMe Cross-Controller Reset Status */ +enum { + NVME_CCR_STATUS_IN_PROGRESS, + NVME_CCR_STATUS_SUCCESS, + NVME_CCR_STATUS_FAILED, +}; + +/* NVMe Cross-Controller Reset Flags */ +enum { + NVME_CCR_FLAGS_VALIDATED = 0x01, + NVME_CCR_FLAGS_INITIATED = 0x02, +}; + +struct nvme_ccr_log_entry { + __le16 icid; + __u8 ciu; + __u8 rsvd3; + __le16 acid; + __u8 ccrs; + __u8 ccrf; +}; + +struct nvme_ccr_log { + __le16 ne; + __u8 rsvd2[6]; + struct nvme_ccr_log_entry entries[NVMF_CCR_PER_PAGE]; +}; + /* NVMe Namespace Write Protect State */ enum { NVME_NS_NO_WRITE_PROTECT = 0, @@ -1538,6 +1599,22 @@ struct nvme_abort_cmd { __u32 rsvd11[5]; }; +struct nvme_cancel_cmd { + __u8 opcode; + __u8 flags; + __u16 command_id; + __le32 nsid; + __u32 rsvd1[8]; + __le16 sqid; + __u16 cid; + __u8 action; + __u8 rsvd11[3]; + __u32 rsvd12[4]; +}; + +#define NVME_CANCEL_ACTION_MUL_CMD 1 +#define NVME_CANCEL_ACTION_SINGLE_CMD 0 + struct nvme_download_firmware { __u8 opcode; __u8 flags; @@ -2005,6 +2082,7 @@ struct nvme_command { struct nvme_zone_mgmt_send_cmd zms; struct nvme_zone_mgmt_recv_cmd zmr; struct nvme_abort_cmd abort; + struct nvme_cancel_cmd cancel; struct nvme_get_log_page_command get_log_page; struct nvmf_common_command fabrics; struct nvmf_connect_command connect; @@ -2016,6 +2094,7 @@ struct nvme_command { struct nvme_dbbuf dbbuf; struct nvme_directive_cmd directive; struct nvme_io_mgmt_recv_cmd imr; + struct nvme_cross_ctrl_reset_cmd ccr; }; }; @@ -2180,6 +2259,9 @@ enum { NVME_SC_PMR_SAN_PROHIBITED = 0x123, NVME_SC_ANA_GROUP_ID_INVALID = 0x124, NVME_SC_ANA_ATTACH_FAILED = 0x125, + NVME_SC_CCR_IN_PROGRESS = 0x13f, + NVME_SC_CCR_LOGPAGE_FULL = 0x140, + NVME_SC_CCR_LIMIT_EXCEEDED = 0x141, /* * I/O Command Set Specific - NVM commands: @@ -2188,6 +2270,7 @@ enum { NVME_SC_INVALID_PI = 0x181, NVME_SC_READ_ONLY = 0x182, NVME_SC_CMD_SIZE_LIM_EXCEEDED = 0x183, + NVME_SC_INVALID_CID = 0x184, /* * I/O Command Set Specific - Fabrics commands: