From 70c0b0fc6667a22058666a463cbc5604dcddb10a Mon Sep 17 00:00:00 2001
From: OCBender <181250370+OCBender@users.noreply.github.com>
Date: Fri, 18 Sep 2026 12:21:37 -0400
Subject: [PATCH 1/4] fix aha.svc.mirror bugs

---
 changes/97be8d1fc0995d182d1d578fa598ca22.yaml |   8 +
 changes/a0042c7e6f67dc0c4915db5c390acf1d.yaml |   9 +
 changes/ccaf3521f3a56eb95f109570850e0c64.yaml |  12 +
 changes/ffcb3ca2016c562e1439f12605b64bf2.yaml |   8 +
 docs/synapse/userguides/storm_ref_cmd.rstorm  |  13 +
 synapse/lib/stormlib/aha.py                   | 301 +++++++++++---
 synapse/tests/test_lib_stormlib_aha.py        | 123 ++++++
 synapse/tests/test_tools_aha.py               |  45 ++-
 synapse/tests/utils.py                        |   2 +-
 synapse/tools/aha/mirror.py                   | 378 +++++++++++++-----
 10 files changed, 722 insertions(+), 177 deletions(-)
 create mode 100644 changes/97be8d1fc0995d182d1d578fa598ca22.yaml
 create mode 100644 changes/a0042c7e6f67dc0c4915db5c390acf1d.yaml
 create mode 100644 changes/ccaf3521f3a56eb95f109570850e0c64.yaml
 create mode 100644 changes/ffcb3ca2016c562e1439f12605b64bf2.yaml

diff --git a/changes/97be8d1fc0995d182d1d578fa598ca22.yaml b/changes/97be8d1fc0995d182d1d578fa598ca22.yaml
new file mode 100644
index 00000000000..8f68ea24da7
--- /dev/null
+++ b/changes/97be8d1fc0995d182d1d578fa598ca22.yaml
@@ -0,0 +1,8 @@
+---
+desc: Fixed an issue where the ``synapse.tests.utils.TstOutPut.expect()`` test helper
+  ignored the caller's string when called with ``whitespace=False``, causing those
+  assertions to always pass.
+desc:literal: false
+prs: []
+type: bug
+...
diff --git a/changes/a0042c7e6f67dc0c4915db5c390acf1d.yaml b/changes/a0042c7e6f67dc0c4915db5c390acf1d.yaml
new file mode 100644
index 00000000000..17f6b1b2dd1
--- /dev/null
+++ b/changes/a0042c7e6f67dc0c4915db5c390acf1d.yaml
@@ -0,0 +1,9 @@
+---
+desc: Fixed an issue where the ``aha.svc.mirror`` Storm command and the ``synapse.tools.aha.mirror``
+  tool reported a mirror group as being in sync when no member of the group responded,
+  and where a service which responded under its leader alias name was reported with
+  an unknown status.
+desc:literal: false
+prs: []
+type: bug
+...
diff --git a/changes/ccaf3521f3a56eb95f109570850e0c64.yaml b/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
new file mode 100644
index 00000000000..29ac94aa849
--- /dev/null
+++ b/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
@@ -0,0 +1,12 @@
+---
+desc: Updated the ``aha.svc.mirror`` Storm command and the ``synapse.tools.aha.mirror``
+  tool to identify mirror group members by their shared cell iden. Mirror groups are
+  now displayed even when no service has claimed the group leader name. Added a ``follows``
+  column which displays the service that each member mirrors from, and a ``Group Leader``
+  line which displays the service currently reporting itself as the leader. The two
+  now produce identical output, and both report ``No mirror groups found.`` when there
+  are no mirror groups to display.
+desc:literal: false
+prs: []
+type: feat
+...
diff --git a/changes/ffcb3ca2016c562e1439f12605b64bf2.yaml b/changes/ffcb3ca2016c562e1439f12605b64bf2.yaml
new file mode 100644
index 00000000000..7865919a301
--- /dev/null
+++ b/changes/ffcb3ca2016c562e1439f12605b64bf2.yaml
@@ -0,0 +1,8 @@
+---
+desc: Fixed an issue where the ``aha.svc.mirror`` Storm command and the ``synapse.tools.aha.mirror``
+  tool omitted mirror group members which had not registered themselves under the
+  group leader name.
+desc:literal: false
+prs: []
+type: bug
+...
diff --git a/docs/synapse/userguides/storm_ref_cmd.rstorm b/docs/synapse/userguides/storm_ref_cmd.rstorm
index abdf39d7ccb..b07e51b04d9 100644
--- a/docs/synapse/userguides/storm_ref_cmd.rstorm
+++ b/docs/synapse/userguides/storm_ref_cmd.rstorm
@@ -175,6 +175,7 @@ service pools.
 - `aha.pool.svc.add`_
 - `aha.pool.svc.del`_
 - `aha.svc.list`_
+- `aha.svc.mirror`_
 - `aha.svc.stat`_
 
 Help for individual ``aha.*`` commands can be displayed using:
@@ -253,6 +254,18 @@ The ``aha.svc.list`` command lists AHA services.
 .. storm-cli:: aha.svc.list --help
 
 
+.. _storm-aha-svc-mirror:
+
+aha.svc.mirror
+++++++++++++++
+
+The ``aha.svc.mirror`` command displays AHA mirror groups and the status of their members.
+
+**Syntax:**
+
+.. storm-cli:: aha.svc.mirror --help
+
+
 .. _storm-aha-svc-stat:
 
 aha.svc.stat
diff --git a/synapse/lib/stormlib/aha.py b/synapse/lib/stormlib/aha.py
index ea5b4d86ff5..4cbafcf6c76 100644
--- a/synapse/lib/stormlib/aha.py
+++ b/synapse/lib/stormlib/aha.py
@@ -674,7 +674,17 @@ async def _methPoolSvcDel(self, svcname):
         'descr': textwrap.dedent('''\
             Query the AHA services and their mirror relationships.
 
-            Note: non-mirror services are not displayed.
+            Mirror group members are identified by the cell iden which they share, so a
+            group is displayed even when no service has claimed the group leader name.
+
+            The role and follows columns reflect the status reported by each service,
+            where follows is the service which that member mirrors from.
+
+            Notes:
+                - Non-mirror services are not displayed, nor are groups which have no
+                  online service.
+                - A service restored from another service's backup shares its cell iden
+                  and is displayed as a member of that group.
         '''),
         'cmdargs': (
             ('--timeout', {'help': 'The timeout in seconds for individual service API calls.',
@@ -695,6 +705,7 @@ async def _methPoolSvcDel(self, svcname):
                     {"name": "version", "width": 12},
                     {"name": "synapse", "width": 12},
                     {"name": "nexus idx", "width": 10},
+                    {"name": "follows"},
                 ],
                 "separators": {
                     "row:outline": false,
@@ -709,17 +720,32 @@ async def _methPoolSvcDel(self, svcname):
             $wait = $cmdopts.wait
         }
 
-        function get_cell_infos(vname, timeout) {
-            $cell_infos = ({})
+        // AHA de-duplicates peer responses by run iden and labels each with whichever
+        // service entry it resolved first, which may be the leader alias rather than the
+        // member entry. Key responses by run iden and fall back to the name for peers
+        // which did not report one.
+        function get_cell_infos(svcname, timeout) {
+            $byrun = ({})
+            $byname = ({})
             $todo = $lib.utils.todo('getCellInfo')
-            for $info in $lib.aha.callPeerApi($vname, $todo, timeout=$timeout) {
-                $svcname = $info.0
-                ($ok, $info) = $info.1
-                if $ok {
-                    $cell_infos.$svcname = $info
+            try {
+                for $item in $lib.aha.callPeerApi($svcname, $todo, timeout=$timeout) {
+                    $peername = $item.0
+                    ($ok, $info) = $item.1
+                    if (not $ok) { continue }
+
+                    $byname.$peername = $info
+
+                    $run = $info.cell.run
+                    if ($run != null) {
+                        $byrun.$run = $info
+                    }
                 }
+            } catch * as err {
+                $lib.warn(`Failed to query mirror group members for {$svcname}: {$err.mesg}`)
             }
-            return($cell_infos)
+
+            return(({"byrun": $byrun, "byname": $byname}))
         }
 
         function build_status_list(members, cell_infos) {
@@ -729,6 +755,7 @@ async def _methPoolSvcDel(self, svcname):
                 $svcname = $svc.name
                 $status = ({
                     'name': $svcname,
+                    'responded': (false),
                     'role': '<unknown>',
                     'online': $lib.dict.has($svcinfo, 'online'),
                     'ready': $svcinfo.ready,
@@ -736,10 +763,24 @@ async def _methPoolSvcDel(self, svcname):
                     'port': $svcinfo.urlinfo.port,
                     'version': '<unknown>',
                     'synapse_version': '<unknown>',
-                    'nexs_indx': (0)
+                    'nexs_indx': (null),
+                    'follows': '<unknown>'
                 })
-                if ($cell_infos.$svcname) {
-                    $info = $cell_infos.$svcname
+                $info = (null)
+
+                $byrun = $cell_infos.byrun
+                $run = $svcinfo.run
+                if ($run != null) {
+                    $info = $byrun.$run
+                }
+
+                if ($info = null) {
+                    $byname = $cell_infos.byname
+                    $info = $byname.$svcname
+                }
+
+                if ($info != null) {
+                    $status.responded = (true)
                     $cell_info = $info.cell
                     $status.nexs_indx = $cell_info.nexsindx
                     if ($cell_info.active) {
@@ -749,32 +790,93 @@ async def _methPoolSvcDel(self, svcname):
                     }
                     $status.version = $info.cell.verstring
                     $status.synapse_version = $info.synapse.verstring
+
+                    // The URL is sanitized by the service before it is returned to us.
+                    if $lib.dict.has($cell_info, 'mirror') {
+                        $mirror = $cell_info.mirror
+                        if ($mirror = null) {
+                            $status.follows = '<none - write root>'
+                        } else {
+                            $status.follows = $mirror
+                        }
+                    }
                 }
                 $group_status.append($status)
             }
             return($group_status)
         }
 
-        function check_sync_status(group_status) {
+        function get_sync_status(group_status) {
+
             $indices = $lib.set()
-            $known_count = (0)
+            $known = (0)
+
             for $status in $group_status {
-                $indices.add($status.nexs_indx)
-                $known_count = ($known_count + (1))
+                $indx = $status.nexs_indx
+                if ($indx = null) {
+                    continue
+                }
+                $indices.add($indx)
+                $known = ($known + (1))
+            }
+
+            // No member reported an index, so we can say nothing about replication.
+            if ($known = 0) {
+                return('Unknown')
             }
-            if ($lib.len($indices) = 1) {
-                if ($known_count = $lib.len($group_status)) {
-                    return(true)
+
+            if (($known = $lib.len($group_status)) and ($lib.len($indices) = 1)) {
+                return('In Sync')
+            }
+
+            return('Out of Sync')
+        }
+
+        // Leadership is reported by the service itself, so it stays accurate when the
+        // leader alias is missing or stale.
+        function get_group_leaders(group_status) {
+            $leaders = ()
+            for $status in $group_status {
+                if ($status.role = 'leader') {
+                    $leaders.append($status.name)
                 }
             }
+            return($leaders)
+        }
+
+        function output_group_leaders(leaders) {
+            if ($lib.len($leaders) = 0) {
+                $lib.print('Group Leader: <none>')
+            } elif ($lib.len($leaders) = 1) {
+                $lib.print(`Group Leader: {$leaders.0}`)
+            } else {
+                $lib.print(`Group Leader: <multiple: {(', ').join($leaders)}>`)
+            }
+        }
+
+        function get_group_name(iden, alias, members) {
+
+            if ($alias != null) {
+                return($alias.name)
+            }
+
+            for $svc in $members {
+                $leader = $svc.svcinfo.leader
+                if ($leader != null) {
+                    return(`{$leader}.{$svc.svcnetw} (leader alias not registered)`)
+                }
+            }
+
+            return(`<no leader alias> (cell iden: {$iden})`)
         }
 
         function output_status(vname, group_status, printer) {
             $lib.print($printer.header())
             $lib.print($vname)
             for $status in $group_status {
-                if ($status.nexs_indx = 0) {
-                    $status.nexs_indx = '<unknown>'
+                $nexs = $status.nexs_indx
+                if ($nexs = null) {
+                    $nexs = '<unknown>'
                 }
                 $row = (
                     $status.name,
@@ -785,97 +887,164 @@ async def _methPoolSvcDel(self, svcname):
                     $status.port,
                     $status.version,
                     $status.synapse_version,
-                    $status.nexs_indx
+                    $nexs,
+                    $status.follows
                 )
                 $lib.print($printer.row($row))
             }
         }
 
-        $virtual_services = ({})
-        $member_servers = ({})
+        // The leader alias is only registered by an active service, so a group with no
+        // claimed leader has no alias entry to group on. Every member of a mirror group
+        // shares the cell iden, so group on that instead.
+        $leader_aliases = ({})
+        $svcs_by_run = ({})
 
         for $svc in $lib.aha.list() {
-            $name = $svc.name
+
             $svcinfo = $svc.svcinfo
-            $urlinfo = $svcinfo.urlinfo
-            $hostname = $urlinfo.hostname
 
-            if ($name != $hostname) {
-                $virtual_services.$name = $svc
-            } else {
-                $member_servers.$name = $svc
+            $iden = $svcinfo.iden
+            $run = $svcinfo.run
+            if (($iden = null) or ($run = null)) {
+                continue
             }
-        }
 
-        $mirror_groups = ({})
-        for ($vname, $vsvc) in $virtual_services {
-            $vsvc_info = $vsvc.svcinfo
-            $vsvc_iden = $vsvc_info.iden
-            $vsvc_leader = $vsvc_info.leader
-            $vsvc_hostname = $vsvc_info.urlinfo.hostname
-
-            if (not $vsvc_iden or not $vsvc_hostname or not $vsvc_leader) {
+            // A service registers its alias with addAhaSvc($leader, $info), so the alias
+            // entry is the one whose service name matches the leader name it carries.
+            $leader = $svcinfo.leader
+            if (($leader != null) and ($svc.svcname = $leader)) {
+                if (not $lib.dict.has($leader_aliases, $iden)) {
+                    $leader_aliases.$iden = $svc
+                }
                 continue
             }
 
-            $primary_member = $member_servers.$vsvc_hostname
-            if (not $primary_member) {
+            // Several entries may share a run iden. Prefer the one which names itself.
+            $runkey = `{$iden}|{$run}`
+            $seen = $svcs_by_run.$runkey
+            if (($seen != null) and ($seen.name = $seen.svcinfo.urlinfo.hostname)) {
                 continue
             }
 
-            $members = ([$primary_member])
-            for ($mname, $msvc) in $member_servers {
-                if ($mname != $vsvc_hostname) {
-                    $msvc_info = $msvc.svcinfo
-                    if ($msvc_info.iden = $vsvc_iden and $msvc_info.leader = $vsvc_leader) {
-                        $members.append($msvc)
-                    }
+            $svcs_by_run.$runkey = $svc
+        }
+
+        $members_by_iden = ({})
+        for ($runkey, $svc) in $svcs_by_run {
+            $iden = $svc.svcinfo.iden
+            $members = $members_by_iden.$iden
+            if ($members = null) {
+                $members = ({})
+            }
+            $name = $svc.name
+            $members.$name = $svc
+            $members_by_iden.$iden = $members
+        }
+
+        $mirror_groups = ()
+        for ($iden, $membermap) in $members_by_iden {
+
+            $names = $lib.dict.keys($membermap)
+            if ($lib.len($names) <= 1) {
+                continue
+            }
+            $names.sort()
+
+            $members = ()
+            $anyonline = (false)
+            for $name in $names {
+                $svc = $membermap.$name
+                $members.append($svc)
+                if $lib.dict.has($svc.svcinfo, 'online') {
+                    $anyonline = (true)
                 }
             }
 
-            if ($lib.len($members) > 1) {
-                $mirror_groups.$vname = $members
+            // AHA never reaps service entries, so a decommissioned cluster lingers in the
+            // registry forever. Only report groups which still have a service online.
+            if (not $anyonline) {
+                continue
             }
+
+            $mirror_groups.append(($iden, $members))
         }
 
-        for ($vname, $members) in $mirror_groups {
-            $cell_infos = $get_cell_infos($vname, $timeout)
-            $group_status = $build_status_list($members, $cell_infos)
+        if ($lib.len($mirror_groups) = 0) {
+            $lib.print('No mirror groups found.')
+        } else {
             $lib.print('Service Mirror Groups:')
+        }
+
+        for ($iden, $members) in $mirror_groups {
+
+            $alias = $leader_aliases.$iden
+            $vname = $get_group_name($iden, $alias, $members)
+
+            // callPeerApi() resolves any group member to the shared cell iden, so a member
+            // name works when no alias is registered.
+            $svcname = $members.0.name
+            if ($alias != null) {
+                $svcname = $alias.name
+            }
+
+            $cell_infos = $get_cell_infos($svcname, $timeout)
+            $group_status = $build_status_list($members, $cell_infos)
             $output_status($vname, $group_status, $printer)
 
-            if $check_sync_status($group_status) {
-                $lib.print('Group Status: In Sync')
-            } else {
-                $lib.print(`Group Status: Out of Sync`)
+            $leaders = $get_group_leaders($group_status)
+            $output_group_leaders($leaders)
+
+            $syncstatus = $get_sync_status($group_status)
+            $lib.print(`Group Status: {$syncstatus}`)
+
+            if ($syncstatus != 'In Sync') {
                 if $wait {
-                    $leader_nexs = (0)
+
+                    $allresp = (true)
                     for $status in $group_status {
-                        if (($status.role = 'leader') and ($status.nexs_indx > 0)) {
+                        if (not $status.responded) {
+                            $allresp = (false)
+                        }
+                    }
+
+                    $leader_nexs = (null)
+                    for $status in $group_status {
+                        if (($status.role = 'leader') and ($status.nexs_indx != null)) {
                             $leader_nexs = $status.nexs_indx
                         }
                     }
-                    if ($leader_nexs > 0) {
+
+                    // Without a single leader there is no replication to wait on, and
+                    // an unresponsive member can never satisfy the wait loop.
+                    if ($lib.len($leaders) != 1) {
+                        $lib.warn('Skipping --wait: the group has no active leader.')
+                    } elif (not $allresp) {
+                        $lib.warn('Skipping --wait: one or more group members did not respond.')
+                    } elif ($leader_nexs = null) {
+                        $lib.warn('Skipping --wait: the leader did not report a nexus index.')
+                    } else {
                         while (true) {
                             $responses = ()
                             $todo = $lib.utils.todo(waitNexsOffs, ($leader_nexs - 1), timeout=$timeout)
-                            for $info in $lib.aha.callPeerApi($vname, $todo, timeout=$timeout) {
-                                $svcname = $info.0
+                            for $info in $lib.aha.callPeerApi($svcname, $todo, timeout=$timeout) {
+                                $peername = $info.0
                                 ($ok, $info) = $info.1
                                 if ($ok and $info) {
-                                    $responses.append(($svcname, $info))
+                                    $responses.append(($peername, $info))
                                 }
                             }
 
                             if ($lib.len($responses) = $lib.len($members)) {
-                                $cell_infos = $get_cell_infos($vname, $timeout)
+                                $cell_infos = $get_cell_infos($svcname, $timeout)
                                 $group_status = $build_status_list($members, $cell_infos)
 
                                 $lib.print('')
                                 $lib.print('Updated status:')
                                 $output_status($vname, $group_status, $printer)
 
-                                if $check_sync_status($group_status) {
+                                $syncstatus = $get_sync_status($group_status)
+                                if ($syncstatus = 'In Sync') {
                                     $lib.print('Group Status: In Sync')
                                     break
                                 }
diff --git a/synapse/tests/test_lib_stormlib_aha.py b/synapse/tests/test_lib_stormlib_aha.py
index ed258a15614..8abd8704157 100644
--- a/synapse/tests/test_lib_stormlib_aha.py
+++ b/synapse/tests/test_lib_stormlib_aha.py
@@ -348,12 +348,43 @@ async def test_stormlib_aha_mirror(self):
                 self.stormIsInPrint('cell.synapse', msgs)
                 self.stormIsInPrint('00.cell.synapse', msgs)
                 self.stormIsInPrint('01.cell.synapse', msgs)
+                self.stormIsInPrint('Group Leader: 00.cell.synapse', msgs)
                 self.stormIsInPrint('Group Status: In Sync', msgs)
 
+                # Each member reports the service it mirrors from.
+                self.stormIsInPrint('follows', msgs)
+                self.stormIsInPrint('00.cell.synapse leader True True', msgs, whitespace=False)
+                self.stormIsInPrint('<none - write root>', msgs)
+                self.stormIsInPrint('aha://root@cell...', msgs)
+
                 msgs = await core00.stormlist('aha.svc.mirror --timeout 30')
                 self.stormIsInPrint('Service Mirror Groups:', msgs)
                 self.stormIsInPrint('Group Status: In Sync', msgs)
 
+                # AHA de-duplicates peer responses by run iden and may label the leader
+                # response with the leader alias name rather than the member name. The
+                # leader must still be attributed to its member row.
+                cell01_rid = (await cell01.getCellInfo())['cell']['run']
+
+                async def mock_alias_named(*args, **kwargs):
+                    yield ('cell.synapse', (True, {
+                        'cell': {'run': cell00_rid, 'ready': True, 'nexsindx': 99, 'active': True,
+                                 'verstring': '2.190.0', 'mirror': None},
+                        'synapse': {'verstring': '2.190.0'},
+                    }))
+                    yield ('01.cell.synapse', (True, {
+                        'cell': {'run': cell01_rid, 'ready': True, 'nexsindx': 99, 'active': False,
+                                 'verstring': '2.190.0', 'mirror': 'aha://root@cell...'},
+                        'synapse': {'verstring': '2.190.0'},
+                    }))
+
+                with mock.patch.object(aha, 'callAhaPeerApi', mock_alias_named):
+                    msgs = await core00.stormlist('aha.svc.mirror')
+                    self.stormIsInPrint('00.cell.synapse leader', msgs, whitespace=False)
+                    self.stormIsInPrint('01.cell.synapse follower', msgs, whitespace=False)
+                    self.stormIsInPrint('Group Status: In Sync', msgs)
+                    self.stormNotInPrint('<unknown>', msgs)
+
                 async def mockCellInfo():
                     return {
                         'cell': {'ready': True, 'nexsindx': 10, 'active': True},
@@ -393,10 +424,102 @@ async def mock_call_aha(*args, **kwargs):
                         msgs = await core00.stormlist('aha.svc.mirror --timeout 1')
                         self.stormIsInPrint('Group Status: Out of Sync', msgs)
 
+                # No member reports as active, so the group has no claimed leader and
+                # there is nothing to wait on.
+                async def mockNoLeaderCellInfo():
+                    return {
+                        'cell': {'ready': True, 'nexsindx': 7, 'active': False},
+                        'synapse': {'verstring': '2.190.0'},
+                    }
+
+                with mock.patch.object(cell00, 'getCellInfo', mockNoLeaderCellInfo):
+                    with mock.patch.object(cell01, 'getCellInfo', mockOutOfSyncCellInfo):
+
+                        msgs = await core00.stormlist('aha.svc.mirror --timeout 1')
+                        self.stormIsInPrint('Group Leader: <none>', msgs)
+                        self.stormIsInPrint('Group Status: Out of Sync', msgs)
+                        self.stormNotInPrint('leader', msgs)
+
+                        msgs = await core00.stormlist('aha.svc.mirror --timeout 1 --wait')
+                        self.stormIsInWarn('Skipping --wait: the group has no active leader.', msgs)
+                        self.stormNotInPrint('Updated status:', msgs)
+
+                # Two services both claiming to be active is a split brain.
+                with mock.patch.object(cell00, 'getCellInfo', mockCellInfo):
+                    with mock.patch.object(cell01, 'getCellInfo', mockCellInfo):
+                        msgs = await core00.stormlist('aha.svc.mirror --timeout 1')
+                        self.stormIsInPrint('Group Leader: <multiple: 00.cell.synapse, 01.cell.synapse>', msgs)
+
+                # A group where no member responded tells us nothing about replication
+                # and must not be reported as in sync.
+                async def mock_all_failed(*args, **kwargs):
+                    yield ('00.cell.synapse', (False, 'error'))
+                    yield ('01.cell.synapse', (False, 'error'))
+
+                with mock.patch.object(aha, 'callAhaPeerApi', mock_all_failed):
+                    msgs = await core00.stormlist('aha.svc.mirror --timeout 1')
+                    self.stormIsInPrint('Group Status: Unknown', msgs)
+                    self.stormIsInPrint('Group Leader: <none>', msgs)
+
+                    msgs = await core00.stormlist('aha.svc.mirror --timeout 1 --wait')
+                    self.stormIsInWarn('Skipping --wait: the group has no active leader.', msgs)
+
+                # An unresponsive member can never satisfy the wait loop.
+                async def mock_partial(*args, **kwargs):
+                    yield ('00.cell.synapse', (True, {
+                        'cell': {'run': cell00_rid, 'ready': True, 'nexsindx': 12, 'active': True,
+                                 'verstring': '2.190.0', 'mirror': None},
+                        'synapse': {'verstring': '2.190.0'},
+                    }))
+
+                with mock.patch.object(aha, 'callAhaPeerApi', mock_partial):
+                    msgs = await core00.stormlist('aha.svc.mirror --timeout 1 --wait')
+                    self.stormIsInWarn('Skipping --wait: one or more group members did not respond.', msgs)
+
                 await cell01.nexsroot.client.fini()
                 msgs = await core00.stormlist('aha.svc.mirror')
                 self.stormIsInPrint('follower', msgs)
 
+                # A service which registered without the group leader name still belongs
+                # to the group, since mirrors share a cell iden.
+                svcinfo = dict((await aha.getAhaSvc('00.cell...'))['svcinfo'])
+                svcinfo.pop('leader', None)
+                svcinfo['run'] = s_common.guid()
+                svcinfo['urlinfo'] = dict(svcinfo['urlinfo'])
+                svcinfo['urlinfo']['hostname'] = '02.cell.synapse'
+                await aha.addAhaSvc('02.cell', info=svcinfo, network='synapse')
+
+                msgs = await core00.stormlist('aha.svc.mirror')
+                self.stormIsInPrint('02.cell.synapse', msgs)
+
+                await aha.delAhaSvc('02.cell', network='synapse')
+
+                # Entries with no cell iden or no run iden cannot be correlated at all.
+                await aha.addAhaSvc('noiden.cell', info={'urlinfo': {'scheme': 'tcp',
+                                                                    'host': '0.0.0.0',
+                                                                    'port': '3030'}},
+                                    network='synapse')
+                norun = dict(svcinfo)
+                norun.pop('run', None)
+                await aha.addAhaSvc('norun.cell', info=norun, network='synapse')
+
+                msgs = await core00.stormlist('aha.svc.mirror')
+                self.stormNotInPrint('noiden.cell.synapse', msgs)
+                self.stormNotInPrint('norun.cell.synapse', msgs)
+
+                await aha.delAhaSvc('norun.cell', network='synapse')
+
+                # The group is still reported when no service has claimed the leader
+                # name, which is when an operator most needs to see it.
+                await aha.delAhaSvc('cell', network='synapse')
+
+                msgs = await core00.stormlist('aha.svc.mirror')
+                self.stormIsInPrint('Service Mirror Groups:', msgs)
+                self.stormIsInPrint('cell.synapse (leader alias not registered)', msgs)
+                self.stormIsInPrint('00.cell.synapse', msgs)
+                self.stormIsInPrint('01.cell.synapse', msgs)
+
                 await aha.delAhaSvc('00.cell', network='synapse')
                 msgs = await core00.stormlist('aha.svc.mirror')
                 self.stormNotInPrint('Service Mirror Groups:', msgs)
+                self.stormIsInPrint('No mirror groups found.', msgs)
diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py
index 2a89da27835..f4d7dab4246 100644
--- a/synapse/tests/test_tools_aha.py
+++ b/synapse/tests/test_tools_aha.py
@@ -41,17 +41,15 @@ async def test_aha_list(self):
                     retn, outp = await self.execToolMain(s_a_list.main, argv)
                     self.eq(retn, 0)
 
-                    outp.expect('''
-                        Service              network                        leader
-                        cell0                synapse                        None
-                        cell1                example.net                    None
-                    ''', whitespace=False)
+                    outp.expect('Service              network                        leader', whitespace=False)
+                    outp.expect('cell0                synapse                        None', whitespace=False)
+                    outp.expect('cell1                synapse                        None', whitespace=False)
 
                     argv = [ahaurl, 'demo.net']
                     retn, outp = await self.execToolMain(s_a_list.main, argv)
                     self.eq(retn, 0)
                     outp.expect('Service              network', whitespace=False)
-                    outp.expect('cell0                demo.net', whitespace=False)
+                    self.notin('cell0', str(outp))
 
         async with self.getTestCore() as core:
             curl = core.getLocalUrl()
@@ -196,7 +194,7 @@ async def test_aha_mirror(self):
                 argv = ['--url', aha.getLocalUrl()]
                 retn, outp = await self.execToolMain(s_a_mirror.main, argv)
                 self.eq(retn, 0)
-                outp.expect('Service Mirror Groups:')
+                outp.expect('No mirror groups found.')
                 self.notin('no.iden', str(outp))
 
             conf_no_host = {'aha:provision': await aha.addAhaSvcProv('no.host')}
@@ -208,7 +206,7 @@ async def test_aha_mirror(self):
                 argv = ['--url', aha.getLocalUrl()]
                 retn, outp = await self.execToolMain(s_a_mirror.main, argv)
                 self.eq(retn, 0)
-                outp.expect('Service Mirror Groups:')
+                outp.expect('No mirror groups found.')
                 self.notin('no.host', str(outp))
 
             conf_no_leader = {'aha:provision': await aha.addAhaSvcProv('no.leader')}
@@ -219,7 +217,7 @@ async def test_aha_mirror(self):
                 argv = ['--url', aha.getLocalUrl()]
                 retn, outp = await self.execToolMain(s_a_mirror.main, argv)
                 self.eq(retn, 0)
-                outp.expect('Service Mirror Groups:')
+                outp.expect('No mirror groups found.')
                 self.notin('no.leader', str(outp))
 
             conf_no_primary = {'aha:provision': await aha.addAhaSvcProv('no.primary')}
@@ -246,8 +244,14 @@ async def test_aha_mirror(self):
             outp.expect('Service Mirror Groups:')
             outp.expect('00.cell.synapse')
             outp.expect('01.cell.synapse')
+            outp.expect('Group Leader: 00.cell.synapse')
             outp.expect('Group Status: In Sync')
 
+            # Each member reports the service it mirrors from.
+            outp.expect('follows')
+            outp.expect('<none - write root>')
+            outp.expect('aha://root@cell...')
+
             argv = ['--url', ahaurl, '--timeout', '30']
             retn, outp = await self.execToolMain(s_a_mirror.main, argv)
             self.eq(retn, 0)
@@ -337,12 +341,25 @@ async def mock_failed_api(*args, **kwargs):
             with mock.patch.object(aha, 'callAhaPeerApi', mock_failed_api):
                 argv = ['--url', ahaurl, '--timeout', '1']
                 retn, outp = await self.execToolMain(s_a_mirror.main, argv)
-                outp.expect('00.cell.synapse                          leader     True     True    127.0.0.1', whitespace=False)
-                outp.expect('nexsindx      10', whitespace=False)
-                outp.expect('02.cell.synapse                          leader     True     True    127.0.0.1', whitespace=False)
-                outp.expect('nexsindx      12', whitespace=False)
+                # None of the mocked responses report themselves as active, and one peer
+                # failed outright, so the group has no leader and cannot be in sync.
+                outp.expect('00.cell.synapse                          follower   True     True    127.0.0.1', whitespace=False)
+                outp.expect('02.cell.synapse                          follower   True     True    127.0.0.1', whitespace=False)
                 outp.expect('01.cell.synapse                          <unknown>  True     True', whitespace=False)
-                outp.expect('<unknown>    <unknown>', whitespace=False)
+                outp.expect('Group Leader: <none>')
+                outp.expect('Group Status: Out of Sync')
+
+            # The group is still reported when no service has claimed the leader name,
+            # which is when an operator most needs to see it.
+            await aha.delAhaSvc('cell', network='synapse')
+
+            argv = ['--url', ahaurl, '--timeout', '1']
+            retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+            self.eq(retn, 0)
+            outp.expect('cell.synapse (leader alias not registered)')
+            outp.expect('00.cell.synapse')
+            outp.expect('01.cell.synapse')
+            outp.expect('02.cell.synapse')
 
         self.eq(s_a_mirror.timeout_type('30'), 30)
         self.eq(s_a_mirror.timeout_type('0'), 0)
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index 386da814ad5..2ce9e779376 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -784,7 +784,7 @@ def expect(self, substr, throw=True, whitespace=True):
 
         if not whitespace:
             outs = ' '.join(outs.split())
-            substr = ' '.join(outs.split())
+            substr = ' '.join(substr.split())
 
         if outs.find(substr) == -1:
             if throw:
diff --git a/synapse/tools/aha/mirror.py b/synapse/tools/aha/mirror.py
index 4bda480dc48..63e531a7357 100644
--- a/synapse/tools/aha/mirror.py
+++ b/synapse/tools/aha/mirror.py
@@ -14,62 +14,308 @@
 
 '''
 
-async def get_cell_infos(prox, iden, members, timeout):
-    cell_infos = {}
-    if iden is not None:
-        todo = s_common.todo('getCellInfo')
+async def get_cell_infos(prox, outp, iden, timeout):
+    '''
+    Aha de-duplicates peer responses by run iden and labels each response with whichever
+    service entry it resolved first, which may be the leader alias rather than the member
+    entry. Key the responses by run iden and fall back to the name for any peer which did
+    not report one.
+    '''
+    byrun = {}
+    byname = {}
+
+    if iden is None:
+        return byrun, byname
+
+    todo = s_common.todo('getCellInfo')
+
+    try:
+
         async for svcname, (ok, info) in prox.callAhaPeerApi(iden, todo, timeout=timeout):
+
             if not ok:
                 continue
-            cell_infos[svcname] = info
-    return cell_infos
 
-def build_status_list(members, cell_infos):
+            byname[svcname] = info
+
+            run = info.get('cell', {}).get('run')
+            if run is not None:
+                byrun[run] = info
+
+    except Exception as e:
+        mesg = repr(e)
+        if isinstance(e, s_exc.SynErr):
+            mesg = e.errinfo.get('mesg', repr(e))
+
+        outp.printf(f'WARNING: Failed to query mirror group members: {mesg}')
+
+    return byrun, byname
+
+def build_status_list(members, byrun, byname):
+
     group_status = []
+
     for svc in members:
+
         svcname = svc.get('name')
         svcinfo = svc.get('svcinfo', {})
+
+        ready = svcinfo.get('ready')
+
         status = {
             'name': svcname,
+            'responded': False,
             'role': '<unknown>',
             'online': str('online' in svcinfo),
-            'ready': 'True',
+            'ready': '' if ready is None else str(ready),
             'host': svcinfo.get('urlinfo', {}).get('host', ''),
             'port': str(svcinfo.get('urlinfo', {}).get('port', '')),
             'version': '<unknown>',
             'synapse': '<unknown>',
-            'nexs_indx': 0
+            'nexs_indx': None,
+            'follows': '<unknown>',
         }
-        if svcname in cell_infos:
-            info = cell_infos[svcname]
+
+        info = byrun.get(svcinfo.get('run'))
+        if info is None:
+            info = byname.get(svcname)
+
+        if info is not None:
+
             cell_info = info.get('cell', {})
+
             status.update({
-                'nexs_indx': cell_info.get('nexsindx', 0),
+                'responded': True,
+                'nexs_indx': cell_info.get('nexsindx'),
                 'role': 'leader' if cell_info.get('active') else 'follower',
-                'version': str(info.get('cell', {}).get('verstring', '')),
+                'version': str(cell_info.get('verstring', '')),
                 'synapse': str(info.get('synapse', {}).get('verstring', '')),
                 'online': 'True',
-                'ready': str(cell_info.get('ready', False))
             })
+
+            # The URL is sanitized by the service before it is returned to us.
+            if 'mirror' in cell_info:
+                mirror = cell_info.get('mirror')
+                status['follows'] = '<none - write root>' if mirror is None else mirror
+
         group_status.append(status)
+
     return group_status
 
+# Mirrors the column configuration used by the aha.svc.mirror Storm command so that both
+# renderings of this data stay identical. A column with no width is not padded.
+columns = (
+    ('name', 40),
+    ('role', 9),
+    ('online', 7),
+    ('ready', 6),
+    ('host', 16),
+    ('port', 8),
+    ('version', 12),
+    ('synapse', 12),
+    ('nexus idx', 10),
+    ('follows', None),
+)
+
+def format_row(values, pad=' '):
+    '''
+    Render a row the way $lib.tabular does with no column separator configured, which is
+    a leading pad, items joined by a doubled pad, and a trailing pad.
+    '''
+    items = []
+    for (name, width), valu in zip(columns, values):
+
+        valu = '' if valu is None else str(valu)
+        if width is not None:
+            valu = valu.ljust(width)
+
+        items.append(valu)
+
+    return f'{pad}{(pad * 2).join(items)}{pad}'
+
 def output_status(outp, vname, group_status):
-    header = ' {:<40} {:<10} {:<8} {:<7} {:<16} {:<9} {:<12} {:<12} {:<10}'.format(
-        'name', 'role', 'online', 'ready', 'host', 'port', 'version', 'synapse', 'nexus idx')
-    outp.printf(header)
-    outp.printf('#' * 120)
+
+    outp.printf(format_row([name for name, width in columns]))
+    outp.printf(format_row([(width or len(name)) * '#' for name, width in columns], pad='#'))
     outp.printf(vname)
+
     for status in group_status:
-        if status['nexs_indx'] == 0:
-            status['nexs_indx'] = '<unknown>'
-        line = ' {name:<40} {role:<10} {online:<8} {ready:<7} {host:<16} {port:<9} {version:<12} {synapse:<12} {nexs_indx:<10}'.format(**status)
-        outp.printf(line)
 
-def check_sync_status(group_status):
-    indices = {status['nexs_indx'] for status in group_status}
-    known_count = sum(1 for status in group_status)
-    return len(indices) == 1 and known_count == len(group_status)
+        nexs = status.get('nexs_indx')
+        if nexs is None:
+            nexs = '<unknown>'
+
+        outp.printf(format_row((
+            status['name'],
+            status['role'],
+            status['online'],
+            status['ready'],
+            status['host'],
+            status['port'],
+            status['version'],
+            status['synapse'],
+            nexs,
+            status['follows'],
+        )))
+
+def get_sync_status(group_status):
+
+    indices = {status['nexs_indx'] for status in group_status if status['nexs_indx'] is not None}
+    known = sum(1 for status in group_status if status['nexs_indx'] is not None)
+
+    # No member reported an index, so we can say nothing about replication.
+    if known == 0:
+        return 'Unknown'
+
+    if known == len(group_status) and len(indices) == 1:
+        return 'In Sync'
+
+    return 'Out of Sync'
+
+def get_group_leaders(group_status):
+    # Leadership is reported by the service itself, so it stays accurate when the leader
+    # alias is missing or stale.
+    return [status['name'] for status in group_status if status['role'] == 'leader']
+
+def output_group_leaders(outp, leaders):
+
+    if not leaders:
+        outp.printf('Group Leader: <none>')
+        return
+
+    if len(leaders) == 1:
+        outp.printf(f'Group Leader: {leaders[0]}')
+        return
+
+    outp.printf(f'Group Leader: <multiple: {", ".join(leaders)}>')
+
+def get_group_name(iden, alias, members):
+
+    if alias is not None:
+        return alias.get('name')
+
+    for svc in members:
+        leader = svc.get('svcinfo', {}).get('leader')
+        if leader is not None:
+            return f'{leader}.{svc.get("svcnetw")} (leader alias not registered)'
+
+    return f'<no leader alias> (cell iden: {iden})'
+
+def get_mirror_groups(svcdefs):
+    '''
+    The leader alias is only registered by an active service, so a group with no claimed
+    leader has no alias entry to group on. Every member of a mirror group shares the cell
+    iden, so group on that instead.
+
+    Returns a tuple of (leader_aliases, mirror_groups).
+    '''
+    leader_aliases = {}
+    svcs_by_run = {}
+
+    for svc in svcdefs:
+
+        svcinfo = svc.get('svcinfo', {})
+
+        iden = svcinfo.get('iden')
+        run = svcinfo.get('run')
+        if iden is None or run is None:
+            continue
+
+        # A service registers its alias with addAhaSvc(leader, info), so the alias entry
+        # is the one whose service name matches the leader name it carries.
+        leader = svcinfo.get('leader')
+        if leader is not None and svc.get('svcname') == leader:
+            leader_aliases.setdefault(iden, svc)
+            continue
+
+        # Several entries may share a run iden. Prefer the one which names itself.
+        seen = svcs_by_run.get((iden, run))
+        if seen is not None:
+            seeninfo = seen.get('svcinfo', {})
+            if seen.get('name') == seeninfo.get('urlinfo', {}).get('hostname'):
+                continue
+
+        svcs_by_run[(iden, run)] = svc
+
+    members_by_iden = {}
+    for (iden, run), svc in svcs_by_run.items():
+        members_by_iden.setdefault(iden, {})[svc.get('name')] = svc
+
+    mirror_groups = []
+    for iden, membermap in members_by_iden.items():
+
+        if len(membermap) <= 1:
+            continue
+
+        members = [membermap[name] for name in sorted(membermap)]
+
+        # Aha never reaps service entries, so a decommissioned cluster lingers in the
+        # registry forever. Only report groups which still have a service online.
+        if not any('online' in svc.get('svcinfo', {}) for svc in members):
+            continue
+
+        mirror_groups.append((iden, members))
+
+    return leader_aliases, mirror_groups
+
+async def output_group(outp, prox, iden, alias, members, opts):
+
+    vname = get_group_name(iden, alias, members)
+
+    byrun, byname = await get_cell_infos(prox, outp, iden, opts.timeout)
+    group_status = build_status_list(members, byrun, byname)
+    output_status(outp, vname, group_status)
+
+    leaders = get_group_leaders(group_status)
+    output_group_leaders(outp, leaders)
+
+    syncstatus = get_sync_status(group_status)
+    outp.printf(f'Group Status: {syncstatus}')
+
+    if syncstatus == 'In Sync' or not opts.wait:
+        return
+
+    allresp = all(status['responded'] for status in group_status)
+
+    leader_nexs = None
+    for status in group_status:
+        if status['role'] == 'leader' and status['nexs_indx'] is not None:
+            leader_nexs = status['nexs_indx']
+
+    # Without a single leader there is no replication to wait on, and an unresponsive
+    # member can never satisfy the wait loop.
+    if len(leaders) != 1:
+        outp.printf('WARNING: Skipping --wait: the group has no active leader.')
+        return
+
+    if not allresp:
+        outp.printf('WARNING: Skipping --wait: one or more group members did not respond.')
+        return
+
+    if leader_nexs is None:
+        outp.printf('WARNING: Skipping --wait: the leader did not report a nexus index.')
+        return
+
+    while True:
+
+        responses = []
+        todo = s_common.todo('waitNexsOffs', leader_nexs - 1, timeout=opts.timeout)
+        async for svcname, (ok, info) in prox.callAhaPeerApi(iden, todo, timeout=opts.timeout):
+            if ok and info:
+                responses.append((svcname, info))
+
+        if len(responses) == len(members):
+
+            byrun, byname = await get_cell_infos(prox, outp, iden, opts.timeout)
+            group_status = build_status_list(members, byrun, byname)
+
+            outp.printf('')
+            outp.printf('Updated status:')
+            output_status(outp, vname, group_status)
+
+            if get_sync_status(group_status) == 'In Sync':
+                outp.printf('Group Status: In Sync')
+                return
 
 def timeout_type(valu):
     try:
@@ -104,78 +350,18 @@ async def main(argv, outp=s_output.stdout):
                     outp.printf(f'Service at {opts.url} is not an Aha server')
                     return 1
 
-                virtual_services, member_servers = {}, {}
-                async for svc in prox.getAhaSvcs():
-                    name = svc.get('name', '')
-                    svcinfo = svc.get('svcinfo', {})
-                    urlinfo = svcinfo.get('urlinfo', {})
-                    hostname = urlinfo.get('hostname', '')
-
-                    if name != hostname:
-                        virtual_services[name] = svc
-                    else:
-                        member_servers[name] = svc
-
-                mirror_groups = {}
-                for vname, vsvc in virtual_services.items():
-                    vsvc_info = vsvc.get('svcinfo', {})
-                    vsvc_iden = vsvc_info.get('iden')
-                    vsvc_leader = vsvc_info.get('leader')
-                    vsvc_hostname = vsvc_info.get('urlinfo', {}).get('hostname', '')
-
-                    if not vsvc_iden or not vsvc_hostname or not vsvc_leader:
-                        continue
-
-                    primary_member = member_servers.get(vsvc_hostname)
-                    if not primary_member:
-                        continue
-
-                    members = [primary_member] + [
-                        msvc for mname, msvc in member_servers.items()
-                        if mname != vsvc_hostname and
-                        msvc.get('svcinfo', {}).get('iden') == vsvc_iden and
-                        msvc.get('svcinfo', {}).get('leader') == vsvc_leader
-                    ]
-
-                    if len(members) > 1:
-                        mirror_groups[vname] = members
+                svcdefs = [svc async for svc in prox.getAhaSvcs()]
+
+                leader_aliases, mirror_groups = get_mirror_groups(svcdefs)
+
+                if not mirror_groups:
+                    outp.printf('No mirror groups found.')
+                    return 0
 
                 outp.printf('Service Mirror Groups:')
-                for vname, members in mirror_groups.items():
-                    iden = members[0].get('svcinfo', {}).get('iden')
-
-                    cell_infos = await get_cell_infos(prox, iden, members, opts.timeout)
-                    group_status = build_status_list(members, cell_infos)
-                    output_status(outp, vname, group_status)
-
-                    if check_sync_status(group_status):
-                        outp.printf('Group Status: In Sync')
-                    else:
-                        outp.printf('Group Status: Out of Sync')
-                        if opts.wait:
-                            leader_nexs = None
-                            for status in group_status:
-                                if status['role'] == 'leader' and isinstance(status['nexs_indx'], int):
-                                    leader_nexs = status['nexs_indx']
-
-                            if leader_nexs is not None:
-                                while True:
-                                    responses = []
-                                    todo = s_common.todo('waitNexsOffs', leader_nexs - 1, timeout=opts.timeout)
-                                    async for svcname, (ok, info) in prox.callAhaPeerApi(iden, todo, timeout=opts.timeout):
-                                        if ok and info:
-                                            responses.append((svcname, info))
-
-                                    if len(responses) == len(members):
-                                        cell_infos = await get_cell_infos(prox, iden, members, opts.timeout)
-                                        group_status = build_status_list(members, cell_infos)
-
-                                        outp.printf('\nUpdated status:')
-                                        output_status(outp, vname, group_status)
-
-                                        if check_sync_status(group_status):
-                                            outp.printf('Group Status: In Sync')
-                                            break
+                for iden, members in mirror_groups:
+                    await output_group(outp, prox, iden, leader_aliases.get(iden), members, opts)
+                    outp.printf('')
 
                 return 0
 

From a31e5b34f9a0d7cdd83bb91ced1902d54b17a83a Mon Sep 17 00:00:00 2001
From: OCBender <181250370+OCBender@users.noreply.github.com>
Date: Fri, 18 Sep 2026 12:41:38 -0400
Subject: [PATCH 2/4] updates

---
 synapse/tests/test_tools_aha.py | 115 ++++++++++++++++++++++++++++++++
 synapse/tools/aha/mirror.py     |   3 -
 2 files changed, 115 insertions(+), 3 deletions(-)

diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py
index f4d7dab4246..53178d6c298 100644
--- a/synapse/tests/test_tools_aha.py
+++ b/synapse/tests/test_tools_aha.py
@@ -349,6 +349,68 @@ async def mock_failed_api(*args, **kwargs):
                 outp.expect('Group Leader: <none>')
                 outp.expect('Group Status: Out of Sync')
 
+            # An error raised while querying the group members must not abort the report.
+            async def mock_raises(*args, **kwargs):
+                raise s_exc.SynErr(mesg='boom')
+                yield
+
+            with mock.patch.object(aha, 'callAhaPeerApi', mock_raises):
+
+                argv = ['--url', ahaurl, '--timeout', '1']
+                retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+                self.eq(retn, 0)
+                outp.expect('WARNING: Failed to query mirror group members: boom')
+
+                # Nothing responded, so we can say nothing about replication.
+                outp.expect('Group Leader: <none>')
+                outp.expect('Group Status: Unknown')
+
+                argv = ['--url', ahaurl, '--timeout', '1', '--wait']
+                retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+                self.eq(retn, 0)
+                outp.expect('WARNING: Skipping --wait: the group has no active leader.')
+
+            # Several services claiming to be active at once is a split brain.
+            async def mock_split_brain(*args, **kwargs):
+                info = {'cell': {'ready': True, 'nexsindx': 10, 'active': True,
+                                 'verstring': '2.190.0', 'mirror': None},
+                        'synapse': {'verstring': '2.190.0'}}
+                yield ('00.cell.synapse', (True, info))
+                yield ('01.cell.synapse', (True, info))
+                yield ('02.cell.synapse', (True, info))
+
+            with mock.patch.object(aha, 'callAhaPeerApi', mock_split_brain):
+                argv = ['--url', ahaurl, '--timeout', '1']
+                retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+                self.eq(retn, 0)
+                outp.expect('Group Leader: <multiple: 00.cell.synapse, 01.cell.synapse, 02.cell.synapse>')
+
+            # A member which never responded can never satisfy the wait loop.
+            async def mock_partial(*args, **kwargs):
+                yield ('00.cell.synapse', (True, {'cell': {'ready': True, 'nexsindx': 10, 'active': True},
+                                                  'synapse': {'verstring': '2.190.0'}}))
+
+            with mock.patch.object(aha, 'callAhaPeerApi', mock_partial):
+                argv = ['--url', ahaurl, '--timeout', '1', '--wait']
+                retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+                self.eq(retn, 0)
+                outp.expect('WARNING: Skipping --wait: one or more group members did not respond.')
+
+            # A leader which did not report a nexus index gives us no offset to wait on.
+            async def mock_noindx(*args, **kwargs):
+                yield ('00.cell.synapse', (True, {'cell': {'ready': True, 'active': True},
+                                                  'synapse': {'verstring': '2.190.0'}}))
+                yield ('01.cell.synapse', (True, {'cell': {'ready': True, 'nexsindx': 5, 'active': False},
+                                                  'synapse': {'verstring': '2.190.0'}}))
+                yield ('02.cell.synapse', (True, {'cell': {'ready': True, 'nexsindx': 5, 'active': False},
+                                                  'synapse': {'verstring': '2.190.0'}}))
+
+            with mock.patch.object(aha, 'callAhaPeerApi', mock_noindx):
+                argv = ['--url', ahaurl, '--timeout', '1', '--wait']
+                retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+                self.eq(retn, 0)
+                outp.expect('WARNING: Skipping --wait: the leader did not report a nexus index.')
+
             # The group is still reported when no service has claimed the leader name,
             # which is when an operator most needs to see it.
             await aha.delAhaSvc('cell', network='synapse')
@@ -378,3 +440,56 @@ async def mock_failed_api(*args, **kwargs):
             retn, outp = await self.execToolMain(s_a_mirror.main, argv)
             self.eq(retn, 1)
             outp.expect('ERROR: Oof')
+
+    async def test_aha_mirror_grouping(self):
+
+        def svcinfo(iden, run, hostname, **kwargs):
+            info = {
+                'iden': iden,
+                'run': run,
+                'urlinfo': {'scheme': 'tcp', 'host': '127.0.0.1', 'port': 0, 'hostname': hostname},
+            }
+            info.update(kwargs)
+            return info
+
+        async with self.getTestAha() as aha:
+
+            argv = ['--url', aha.getLocalUrl(), '--timeout', '1']
+
+            # A group whose members never registered a leader name has no name to
+            # synthesize from, so it is identified by the cell iden instead.
+            await aha.addAhaSvc('00.noldr', info=svcinfo('iden00', 'run00', '00.noldr.synapse', online='x'),
+                                network='synapse')
+            await aha.addAhaSvc('01.noldr', info=svcinfo('iden00', 'run01', '01.noldr.synapse', online='x'),
+                                network='synapse')
+
+            retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+            self.eq(retn, 0)
+            outp.expect('<no leader alias> (cell iden: iden00)')
+            outp.expect('00.noldr.synapse')
+            outp.expect('01.noldr.synapse')
+
+            # Entries which share a run iden are de-duplicated, preferring the entry
+            # whose service name matches its own hostname.
+            await aha.addAhaSvc('00.dupe', info=svcinfo('iden01', 'run10', '00.dupe.synapse', online='x'),
+                                network='synapse')
+            await aha.addAhaSvc('alias.dupe', info=svcinfo('iden01', 'run10', '00.dupe.synapse', online='x'),
+                                network='synapse')
+            await aha.addAhaSvc('01.dupe', info=svcinfo('iden01', 'run11', '01.dupe.synapse', online='x'),
+                                network='synapse')
+
+            retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+            self.eq(retn, 0)
+            outp.expect('00.dupe.synapse')
+            outp.expect('01.dupe.synapse')
+            self.notin('alias.dupe.synapse', str(outp))
+
+            # Aha never reaps service entries, so a group with no online service is a
+            # decommissioned cluster and is not reported.
+            await aha.addAhaSvc('00.gone', info=svcinfo('iden02', 'run20', '00.gone.synapse'), network='synapse')
+            await aha.addAhaSvc('01.gone', info=svcinfo('iden02', 'run21', '01.gone.synapse'), network='synapse')
+
+            retn, outp = await self.execToolMain(s_a_mirror.main, argv)
+            self.eq(retn, 0)
+            self.notin('00.gone.synapse', str(outp))
+            self.notin('01.gone.synapse', str(outp))
diff --git a/synapse/tools/aha/mirror.py b/synapse/tools/aha/mirror.py
index 63e531a7357..c25ab1714a2 100644
--- a/synapse/tools/aha/mirror.py
+++ b/synapse/tools/aha/mirror.py
@@ -24,9 +24,6 @@ async def get_cell_infos(prox, outp, iden, timeout):
     byrun = {}
     byname = {}
 
-    if iden is None:
-        return byrun, byname
-
     todo = s_common.todo('getCellInfo')
 
     try:

From 6469b17c0dd7704b4a6caa1077ea9219276110aa Mon Sep 17 00:00:00 2001
From: OCBender <181250370+OCBender@users.noreply.github.com>
Date: Fri, 18 Sep 2026 18:39:20 +0000
Subject: [PATCH 3/4] updates

---
 changes/ccaf3521f3a56eb95f109570850e0c64.yaml |  2 +-
 synapse/lib/stormlib/aha.py                   | 10 +++++-----
 synapse/tests/test_lib_stormlib_aha.py        |  4 ++--
 synapse/tests/test_tools_aha.py               |  4 ++--
 synapse/tools/aha/mirror.py                   |  2 +-
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/changes/ccaf3521f3a56eb95f109570850e0c64.yaml b/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
index 29ac94aa849..2e4ebaadcfe 100644
--- a/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
+++ b/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
@@ -1,6 +1,6 @@
 ---
 desc: Updated the ``aha.svc.mirror`` Storm command and the ``synapse.tools.aha.mirror``
-  tool to identify mirror group members by their shared cell iden. Mirror groups are
+  tool to identify mirror group members by their shared service iden. Mirror groups are
   now displayed even when no service has claimed the group leader name. Added a ``follows``
   column which displays the service that each member mirrors from, and a ``Group Leader``
   line which displays the service currently reporting itself as the leader. The two
diff --git a/synapse/lib/stormlib/aha.py b/synapse/lib/stormlib/aha.py
index 4cbafcf6c76..d1cc0ba472f 100644
--- a/synapse/lib/stormlib/aha.py
+++ b/synapse/lib/stormlib/aha.py
@@ -674,7 +674,7 @@ async def _methPoolSvcDel(self, svcname):
         'descr': textwrap.dedent('''\
             Query the AHA services and their mirror relationships.
 
-            Mirror group members are identified by the cell iden which they share, so a
+            Mirror group members are identified by the service iden which they share, so a
             group is displayed even when no service has claimed the group leader name.
 
             The role and follows columns reflect the status reported by each service,
@@ -683,7 +683,7 @@ async def _methPoolSvcDel(self, svcname):
             Notes:
                 - Non-mirror services are not displayed, nor are groups which have no
                   online service.
-                - A service restored from another service's backup shares its cell iden
+                - A service restored from another service's backup shares its service iden
                   and is displayed as a member of that group.
         '''),
         'cmdargs': (
@@ -867,7 +867,7 @@ async def _methPoolSvcDel(self, svcname):
                 }
             }
 
-            return(`<no leader alias> (cell iden: {$iden})`)
+            return(`<no leader alias> (service iden: {$iden})`)
         }
 
         function output_status(vname, group_status, printer) {
@@ -896,7 +896,7 @@ async def _methPoolSvcDel(self, svcname):
 
         // The leader alias is only registered by an active service, so a group with no
         // claimed leader has no alias entry to group on. Every member of a mirror group
-        // shares the cell iden, so group on that instead.
+        // shares the service iden, so group on that instead.
         $leader_aliases = ({})
         $svcs_by_run = ({})
 
@@ -981,7 +981,7 @@ async def _methPoolSvcDel(self, svcname):
             $alias = $leader_aliases.$iden
             $vname = $get_group_name($iden, $alias, $members)
 
-            // callPeerApi() resolves any group member to the shared cell iden, so a member
+            // callPeerApi() resolves any group member to the shared service iden, so a member
             // name works when no alias is registered.
             $svcname = $members.0.name
             if ($alias != null) {
diff --git a/synapse/tests/test_lib_stormlib_aha.py b/synapse/tests/test_lib_stormlib_aha.py
index 8abd8704157..843415d7315 100644
--- a/synapse/tests/test_lib_stormlib_aha.py
+++ b/synapse/tests/test_lib_stormlib_aha.py
@@ -481,7 +481,7 @@ async def mock_partial(*args, **kwargs):
                 self.stormIsInPrint('follower', msgs)
 
                 # A service which registered without the group leader name still belongs
-                # to the group, since mirrors share a cell iden.
+                # to the group, since mirrors share a service iden.
                 svcinfo = dict((await aha.getAhaSvc('00.cell...'))['svcinfo'])
                 svcinfo.pop('leader', None)
                 svcinfo['run'] = s_common.guid()
@@ -494,7 +494,7 @@ async def mock_partial(*args, **kwargs):
 
                 await aha.delAhaSvc('02.cell', network='synapse')
 
-                # Entries with no cell iden or no run iden cannot be correlated at all.
+                # Entries with no service iden or no run iden cannot be correlated at all.
                 await aha.addAhaSvc('noiden.cell', info={'urlinfo': {'scheme': 'tcp',
                                                                     'host': '0.0.0.0',
                                                                     'port': '3030'}},
diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py
index 53178d6c298..9c1e8e45152 100644
--- a/synapse/tests/test_tools_aha.py
+++ b/synapse/tests/test_tools_aha.py
@@ -457,7 +457,7 @@ def svcinfo(iden, run, hostname, **kwargs):
             argv = ['--url', aha.getLocalUrl(), '--timeout', '1']
 
             # A group whose members never registered a leader name has no name to
-            # synthesize from, so it is identified by the cell iden instead.
+            # synthesize from, so it is identified by the service iden instead.
             await aha.addAhaSvc('00.noldr', info=svcinfo('iden00', 'run00', '00.noldr.synapse', online='x'),
                                 network='synapse')
             await aha.addAhaSvc('01.noldr', info=svcinfo('iden00', 'run01', '01.noldr.synapse', online='x'),
@@ -465,7 +465,7 @@ def svcinfo(iden, run, hostname, **kwargs):
 
             retn, outp = await self.execToolMain(s_a_mirror.main, argv)
             self.eq(retn, 0)
-            outp.expect('<no leader alias> (cell iden: iden00)')
+            outp.expect('<no leader alias> (service iden: iden00)')
             outp.expect('00.noldr.synapse')
             outp.expect('01.noldr.synapse')
 
diff --git a/synapse/tools/aha/mirror.py b/synapse/tools/aha/mirror.py
index c25ab1714a2..0f1b026ee6b 100644
--- a/synapse/tools/aha/mirror.py
+++ b/synapse/tools/aha/mirror.py
@@ -196,7 +196,7 @@ def get_group_name(iden, alias, members):
         if leader is not None:
             return f'{leader}.{svc.get("svcnetw")} (leader alias not registered)'
 
-    return f'<no leader alias> (cell iden: {iden})'
+    return f'<no leader alias> (service iden: {iden})'
 
 def get_mirror_groups(svcdefs):
     '''

From 1dd89ba6ee5a9c4b293c4daa879ad4035add62e0 Mon Sep 17 00:00:00 2001
From: OCBender <181250370+OCBender@users.noreply.github.com>
Date: Mon, 21 Sep 2026 13:14:53 -0400
Subject: [PATCH 4/4] feedback

---
 changes/ccaf3521f3a56eb95f109570850e0c64.yaml | 5 ++---
 changes/ffcb3ca2016c562e1439f12605b64bf2.yaml | 8 --------
 2 files changed, 2 insertions(+), 11 deletions(-)
 delete mode 100644 changes/ffcb3ca2016c562e1439f12605b64bf2.yaml

diff --git a/changes/ccaf3521f3a56eb95f109570850e0c64.yaml b/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
index 2e4ebaadcfe..be7beb4d53e 100644
--- a/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
+++ b/changes/ccaf3521f3a56eb95f109570850e0c64.yaml
@@ -3,9 +3,8 @@ desc: Updated the ``aha.svc.mirror`` Storm command and the ``synapse.tools.aha.m
   tool to identify mirror group members by their shared service iden. Mirror groups are
   now displayed even when no service has claimed the group leader name. Added a ``follows``
   column which displays the service that each member mirrors from, and a ``Group Leader``
-  line which displays the service currently reporting itself as the leader. The two
-  now produce identical output, and both report ``No mirror groups found.`` when there
-  are no mirror groups to display.
+  line which displays the service currently reporting itself as the leader. Both report
+  ``No mirror groups found.`` when there are no mirror groups to display.
 desc:literal: false
 prs: []
 type: feat
diff --git a/changes/ffcb3ca2016c562e1439f12605b64bf2.yaml b/changes/ffcb3ca2016c562e1439f12605b64bf2.yaml
deleted file mode 100644
index 7865919a301..00000000000
--- a/changes/ffcb3ca2016c562e1439f12605b64bf2.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-desc: Fixed an issue where the ``aha.svc.mirror`` Storm command and the ``synapse.tools.aha.mirror``
-  tool omitted mirror group members which had not registered themselves under the
-  group leader name.
-desc:literal: false
-prs: []
-type: bug
-...