diff --git a/tests/repeated-load-ownership.zsh b/tests/repeated-load-ownership.zsh
index a4d925a..a4d1a2e 100644
--- a/tests/repeated-load-ownership.zsh
+++ b/tests/repeated-load-ownership.zsh
@@ -6,7 +6,10 @@
 # remove everything the plug-in added across its loads and restore what it
 # replaced. When another plug-in took the plug-in's widget or binding over
 # between its loads, the result must stay as it is on next: that case is still
-# open in the issue. Each scenario runs in its own clean shell.
+# open in the issue. A change made between the loads without zi records (by
+# the user, or by a plug-in loaded in light mode), to a function, widget or
+# binding, is a known limit there as well. Each scenario runs in its own clean
+# shell.
 
 builtin emulate -R zsh
 setopt pipe_fail
@@ -137,6 +140,63 @@ case $ZI_TEST_CASE in
     zi unload "$p" >/dev/null 2>&1
     check '[[ "$(bindkey "^X^P")" == "\"^X^P\" end-of-line" ]]' "the binding differs from next: $(bindkey '^X^P')" || return 1
     ;;
+  linked-taken)
+    # The same, with main linked to viins and the key rebound through
+    # `bindkey -M viins`: a binding in the keymap main is linked to is the
+    # binding of main, so this is the same takeover (z-shell/zi#583).
+    bindkey -v
+    zi load "$p" >/dev/null 2>&1
+    zi load "${ZI_TEST_ROOT}/tv" >/dev/null 2>&1
+    check '[[ "$(bindkey -M viins "^X^P")" == "\"^X^P\" end-of-line" ]]' "tv did not rebind the key: $(bindkey -M viins '^X^P')" || return 1
+    zi load "$p" >/dev/null 2>&1
+    zi unload "$p" >/dev/null 2>&1
+    check '[[ "$(bindkey -M viins "^X^P")" == "\"^X^P\" end-of-line" ]]' "the viins binding differs from next: $(bindkey -M viins '^X^P')" || return 1
+    check '[[ "$(bindkey "^X^P")" == "\"^X^P\" end-of-line" ]]' "the main binding differs from next: $(bindkey '^X^P')" || return 1
+    ;;
+  linked-taken-emacs)
+    # The same with main linked to emacs and `bindkey -M emacs`.
+    bindkey -e
+    zi load "$p" >/dev/null 2>&1
+    zi load "${ZI_TEST_ROOT}/te" >/dev/null 2>&1
+    check '[[ "$(bindkey -M emacs "^X^P")" == "\"^X^P\" end-of-line" ]]' "te did not rebind the key: $(bindkey -M emacs '^X^P')" || return 1
+    zi load "$p" >/dev/null 2>&1
+    zi unload "$p" >/dev/null 2>&1
+    check '[[ "$(bindkey -M emacs "^X^P")" == "\"^X^P\" end-of-line" ]]' "the emacs binding differs from next: $(bindkey -M emacs '^X^P')" || return 1
+    ;;
+  keymap-twice)
+    # A plug-in that creates a keymap, loaded twice: unload deletes it once,
+    # without errors (z-shell/zi#583).
+    typeset out="${ZI_TEST_ROOT}/${ZI_TEST_CASE}.out"
+    zi load "${ZI_TEST_ROOT}/n" >/dev/null 2>&1
+    zi load "${ZI_TEST_ROOT}/n" >/dev/null 2>&1
+    check '(( ${${(f)"$(bindkey -l)"}[(Ie)nmap]} ))' "the loads did not create the keymap" || return 1
+    zi unload "${ZI_TEST_ROOT}/n" >"$out" 2>&1
+    check '(( ! ${${(f)"$(bindkey -l)"}[(Ie)nmap]} ))' "the keymap survived unload" || return 1
+    check '[[ "$(<$out)" != *"no such keymap"* ]]' "unload replayed a keymap record twice: $(<$out)" || return 1
+    typeset -a deleted
+    deleted=( ${(M)${(f)"$(<$out)"}:#Deleting keymap*} )
+    check '(( ${#deleted} == 1 ))' "unload did not delete the keymap exactly once: $(<$out)" || return 1
+    ;;
+  hook-twice)
+    # A plug-in that installs a zle hook widget, loaded twice: unload removes
+    # the hook once, without errors.
+    typeset out="${ZI_TEST_ROOT}/${ZI_TEST_CASE}.out"
+    zi load "${ZI_TEST_ROOT}/h" >/dev/null 2>&1
+    zi load "${ZI_TEST_ROOT}/h" >/dev/null 2>&1
+    check '[[ ${widgets[zle-line-init]} == user:h_fn ]]' "the loads did not install the hook: ${widgets[zle-line-init]}" || return 1
+    zi unload "${ZI_TEST_ROOT}/h" >"$out" 2>&1
+    check '[[ "$(<$out)" != *"no such widget"* ]]' "unload replayed a hook record twice: $(<$out)" || return 1
+    ;;
+  hook-twice-ksh)
+    # As hook-twice, with ksh_arrays set while the plug-in runs `zle -N`:
+    # the record check must not depend on the plug-in's options.
+    typeset out="${ZI_TEST_ROOT}/${ZI_TEST_CASE}.out"
+    zi load "${ZI_TEST_ROOT}/hk" >/dev/null 2>&1
+    zi load "${ZI_TEST_ROOT}/hk" >/dev/null 2>&1
+    check '[[ ${widgets[zle-line-init]} == user:hk_fn ]]' "the loads did not install the hook: ${widgets[zle-line-init]}" || return 1
+    zi unload "${ZI_TEST_ROOT}/hk" >"$out" 2>&1
+    check '[[ "$(<$out)" != *"no such widget"* ]]' "unload replayed a hook record twice: $(<$out)" || return 1
+    ;;
   prior)
     # A function the user defined before any load is not the plug-in's.
     pa_fn() { builtin print -r -- user; }
@@ -202,10 +262,15 @@ ZSH
 
 typeset -i failures=0
 typeset scenario
-for scenario in twice changed interleaved interleaved-other-order older binding-taken main-taken prior wrapped shared shared-twice reload; do
+for scenario in twice changed interleaved interleaved-other-order older binding-taken main-taken linked-taken linked-taken-emacs keymap-twice hook-twice hook-twice-ksh prior wrapped shared shared-twice reload; do
   write_plugin p pa_fn
   write_plugin q qa_fn
-  command mkdir -p "${temp_root}/r" "${temp_root}/k" "${temp_root}/m" "${temp_root}/w" || fail "create the r, k, m and w plug-in directories"
+  command mkdir -p "${temp_root}"/{r,k,m,tv,te,n,h,hk,w} || fail "create the r, k, m, tv, te, n, h, hk and w plug-in directories"
+  builtin print -r -- "bindkey -M viins '^X^P' end-of-line" > "${temp_root}/tv/tv.plugin.zsh" || fail "write the tv plug-in"
+  builtin print -r -- "bindkey -M emacs '^X^P' end-of-line" > "${temp_root}/te/te.plugin.zsh" || fail "write the te plug-in"
+  builtin print -rl -- 'bindkey -N nmap emacs' "bindkey -M nmap '^X^P' end-of-line" > "${temp_root}/n/n.plugin.zsh" || fail "write the n plug-in"
+  builtin print -rl -- 'h_fn() { :; }' 'zle -N zle-line-init h_fn' > "${temp_root}/h/h.plugin.zsh" || fail "write the h plug-in"
+  builtin print -rl -- 'setopt ksh_arrays' 'hk_fn() { :; }' 'zle -N zle-line-init hk_fn' 'unsetopt ksh_arrays' > "${temp_root}/hk/hk.plugin.zsh" || fail "write the hk plug-in"
   builtin print -r -- 'shared_fn() { :; }' > "${temp_root}/r/r.plugin.zsh" || fail "write the r plug-in"
   builtin print -r -- "bindkey '^X^P' end-of-line" > "${temp_root}/k/k.plugin.zsh" || fail "write the k plug-in"
   builtin print -r -- "bindkey -M main '^X^P' end-of-line" > "${temp_root}/m/m.plugin.zsh" || fail "write the m plug-in"
diff --git a/zi.zsh b/zi.zsh
index fc880e4..b0561b4 100644
--- a/zi.zsh
+++ b/zi.zsh
@@ -658,7 +658,7 @@ builtin setopt no_aliases
     fi
     quoted="${(q)quoted}"
     # Remember the bindkey, only when load is in progress (it can be dstart that leads execution here).
-    [[ -n ${ZI[CUR_USPL2]} ]] && ZI[BINDKEYS__${ZI[CUR_USPL2]}]+="$quoted "
+    [[ -n ${ZI[CUR_USPL2]} ]] && .zi-add-record BINDKEYS "$quoted"
     # Remember for dtrace.
     [[ ${ZI[DTRACE]} = 1 ]] && ZI[BINDKEYS___dtrace/_dtrace]+="$quoted "
   else
@@ -675,7 +675,7 @@ builtin setopt no_aliases
       local quoted="${(q)keys} ${(q)widget} ${(q)prev} ${(q)optA} ${(q)mapname} ${(q)optR}"
       quoted="${(q)quoted}"
       # Remember the bindkey, only when load is in progress (it can be dstart that leads execution here).
-      [[ -n ${ZI[CUR_USPL2]} ]] && ZI[BINDKEYS__${ZI[CUR_USPL2]}]+="$quoted "
+      [[ -n ${ZI[CUR_USPL2]} ]] && .zi-add-record BINDKEYS "$quoted"
       [[ ${ZI[DTRACE]} = 1 ]] && ZI[BINDKEYS___dtrace/_dtrace]+="$quoted "
       .zi-add-report "${ZI[CUR_USPL2]}" "Warning: keymap \`main' copied to \`${name}' because of \`${pos[-2]}' substitution"
     # bindkey -N newkeymap [other].
@@ -686,7 +686,7 @@ builtin setopt no_aliases
       local quoted="${(q)keys} ${(q)widget} ${(q)prev} ${(q)optN} ${(q)mapname} ${(q)optR}"
       quoted="${(q)quoted}"
       # Remember the bindkey, only when load is in progress (it can be dstart that leads execution here).
-      [[ -n ${ZI[CUR_USPL2]} ]] && ZI[BINDKEYS__${ZI[CUR_USPL2]}]+="$quoted "
+      [[ -n ${ZI[CUR_USPL2]} ]] && .zi-add-record BINDKEYS "$quoted"
       [[ ${ZI[DTRACE]} = 1 ]] && ZI[BINDKEYS___dtrace/_dtrace]+="$quoted "
     else
       .zi-add-report "${ZI[CUR_USPL2]}" "Warning: last bindkey used non-typical options: ${(kv)opts[*]}"
@@ -786,7 +786,7 @@ builtin setopt no_aliases
       local quoted="$2"
       quoted="${(q)quoted}"
       # Remember only when load is in progress (it can be dstart that leads execution here).
-      [[ -n ${ZI[CUR_USPL2]} ]] && ZI[WIDGETS_DELETE__${ZI[CUR_USPL2]}]+="$quoted "
+      [[ -n ${ZI[CUR_USPL2]} ]] && .zi-add-record WIDGETS_DELETE "$quoted"
       # Remember for dtrace.
       [[ ${ZI[DTRACE]} = 1 ]] && ZI[WIDGETS_DELETE___dtrace/_dtrace]+="$quoted "
     # These will be saved and restored.
@@ -811,7 +811,7 @@ builtin setopt no_aliases
       local quoted="$2"
       quoted="${(q)quoted}"
       # Remember only when load is in progress (it can be dstart that leads execution here).
-      [[ -n ${ZI[CUR_USPL2]} ]] && ZI[WIDGETS_DELETE__${ZI[CUR_USPL2]}]+="$quoted "
+      [[ -n ${ZI[CUR_USPL2]} ]] && .zi-add-record WIDGETS_DELETE "$quoted"
       # Remember for dtrace.
       [[ ${ZI[DTRACE]} = 1 ]] && ZI[WIDGETS_DELETE___dtrace/_dtrace]+="$quoted "
     fi
@@ -1195,6 +1195,9 @@ builtin setopt no_aliases
     # load added join the owned set, and the widget and bindkey records stay.
     # When another plugin took one of those widgets or bindings over in
     # between, the records start again as before; that case remains open.
+    # A change made without zi records (by the user, or by a plugin loaded
+    # in light mode), to a function, widget or binding, is not detected:
+    # unload removes or restores it as if it were this plugin's.
     .zi-keep-previous-load-functions "$uspl2"
   else
     ZI[FUNCTIONS_OWNED__$uspl2]=
@@ -1234,9 +1237,9 @@ builtin setopt no_aliases
 } # ]]]
 # FUNCTION: .zi-repeat-object-key. [[[
 # Sets REPLY to a comparable identity for record $2 of kind $1 (widget-saved,
-# widget-delete or bindkey): "widget NAME", or "bindkey MAP KEY" with an empty
-# MAP for the default keymap, also when it is named main. Fails for a record
-# without such an object.
+# widget-delete or bindkey): "widget NAME", or "bindkey MAP KEY". The default
+# keymap, also when it is named main, is given as $main_map: the keymap main
+# is linked to, set by the caller. Fails for a record without such an object.
 .zi-repeat-object-key() {
   local kind="$1" entry="$2" map
   local -a fields
@@ -1250,16 +1253,30 @@ builtin setopt no_aliases
     bindkey)
       fields=( "${(z)${(Q)entry}}" )
       [[ ${(Q)fields[4]} == -[AN] ]] && return 1
-      # main is the default keymap: `bindkey -M main KEY` and `bindkey KEY`
-      # change the same binding, so they share one identity.
+      # main is the default keymap and an alias of another keymap (`bindkey
+      # -v` links it to viins, `bindkey -e` to emacs): `bindkey KEY`,
+      # `bindkey -M main KEY` and `bindkey -M viins KEY` then change the
+      # same binding, so they share one identity (z-shell/zi#583).
       map=
       [[ ${(Q)fields[4]} == -M ]] && map=${(Q)fields[5]}
-      [[ $map == main ]] && map=
+      [[ -z $map || $map == main ]] && map=$main_map
       REPLY="bindkey $map ${fields[1]}" ;;
     *) return 1 ;;
   esac
   [[ -n $REPLY ]]
 } # ]]]
+# FUNCTION: .zi-add-record. [[[
+# Appends the quoted record $2 to the $1 records (BINDKEYS or WIDGETS_DELETE)
+# of the plugin being loaded, unless they already hold it: a repeated load
+# keeps the earlier load's records, and unload must replay each one once
+# (z-shell/zi#583).
+.zi-add-record() {
+  builtin emulate -L zsh ${=${options[xtrace]:#off}:+-o xtrace}
+  local key="$1__${ZI[CUR_USPL2]}"
+  local -a held
+  held=( "${(z)ZI[$key]}" )
+  (( ${held[(Ie)$2]} )) || ZI[$key]+="$2 "
+} # ]]]
 # FUNCTION: .zi-repeat-objects. [[[
 # Sets reply to the widget and bindkey identities plugin $1 has recorded.
 .zi-repeat-objects() {
@@ -1281,8 +1298,11 @@ builtin setopt no_aliases
 .zi-repeat-taken-over() {
   builtin emulate -LR zsh ${=${options[xtrace]:#off}:+-o xtrace}
   builtin setopt extended_glob
-  local uspl2="$1" key other REPLY
-  local -a reply mine shared
+  local uspl2="$1" key other REPLY main_map=main
+  local -a reply mine shared link
+  # The keymap main is linked to now, for .zi-repeat-object-key.
+  link=( ${(z)"$(builtin bindkey -lL main 2>/dev/null)"} )
+  [[ ${link[2]} == -A && ${link[4]} == main ]] && main_map=${(Q)link[3]}
   .zi-repeat-objects "$uspl2"
   mine=( "${reply[@]}" )
   (( ${#mine} )) || return 1