From 8fcbcc85360bdc01b732f85d54823b153ddb4e22 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 21 Sep 2026 14:49:58 +0000 Subject: [PATCH 1/2] Fix stats.countby --by-name sort with mixed value types (SYN-9957) The --by-name sort key returned int() for names which parsed as integers and the original str for names which did not, so a tally containing both handed sorted() a non-orderable mix of types and leaked a TypeError. This happened most often when some nodes did not have the tallied property set, which tallies under the name None. Sort numeric names numerically using s_common.hugenum() and non-numeric names lexicographically after them. Using a hugenum rather than int() also sorts fractional names numerically. Non-finite values such as nan are treated as non-numeric since they cannot be ordered. Claude-Session: https://claude.ai/code/session_013pyj3S8NddfycqDWj9uBUS --- changes/92730936fe2f60d077b3b1d9db81bb0f.yaml | 8 +++ synapse/lib/stormlib/stats.py | 26 ++++---- synapse/tests/test_lib_stormlib_stats.py | 66 +++++++++++++++++++ 3 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 changes/92730936fe2f60d077b3b1d9db81bb0f.yaml diff --git a/changes/92730936fe2f60d077b3b1d9db81bb0f.yaml b/changes/92730936fe2f60d077b3b1d9db81bb0f.yaml new file mode 100644 index 0000000000..49d94e8ce1 --- /dev/null +++ b/changes/92730936fe2f60d077b3b1d9db81bb0f.yaml @@ -0,0 +1,8 @@ +--- +desc: Fixed a bug where the ``stats.countby`` Storm command could raise a ``TypeError`` + when using ``--by-name`` to sort a tally which contained a mix of numeric and non-numeric + values. +desc:literal: false +prs: [] +type: bug +... diff --git a/synapse/lib/stormlib/stats.py b/synapse/lib/stormlib/stats.py index 927783ebe9..3842530e32 100644 --- a/synapse/lib/stormlib/stats.py +++ b/synapse/lib/stormlib/stats.py @@ -1,3 +1,4 @@ +import decimal import collections import synapse.exc as s_exc @@ -86,17 +87,20 @@ async def execStormCmd(self, runt, genr): return if byname: - # Try to sort numerically instead of lexicographically - def coerce(indx): - def wrapped(valu): - valu = valu[indx] - try: - return int(valu) - except ValueError: - return valu - return wrapped - - values = list(sorted(counts.items(), key=coerce(0))) + # Sort numeric names numerically, then non-numeric names lexicographically. + def sortkey(item): + try: + huge = s_common.hugenum(item[0]) + except (ValueError, decimal.DecimalException): + return (1, decimal.Decimal(0), item[0]) + + # non-finite values such as nan cannot be ordered + if not huge.is_finite(): + return (1, decimal.Decimal(0), item[0]) + + return (0, huge, '') + + values = list(sorted(counts.items(), key=sortkey)) maxv = max(val[1] for val in values) else: diff --git a/synapse/tests/test_lib_stormlib_stats.py b/synapse/tests/test_lib_stormlib_stats.py index c1b1ce174f..3684f92f63 100644 --- a/synapse/tests/test_lib_stormlib_stats.py +++ b/synapse/tests/test_lib_stormlib_stats.py @@ -136,6 +136,44 @@ 13 | 4 | 26.67% | ######################################## '''.strip() +chartunset_byname = ''' +None | 1 | ######################### + 4 | 1 | ######################### + 1 | 2 | ################################################## + 0 | 1 | ######################### +'''.strip() + +chartunset_rev_byname = ''' + 0 | 1 | ######################### + 1 | 2 | ################################################## + 4 | 1 | ######################### +None | 1 | ######################### +'''.strip() + +chartmixed_byname = ''' + nan | 1 | ################################################## + abc | 1 | ################################################## +None | 1 | ################################################## +0xzz | 1 | ################################################## +0x20 | 1 | ################################################## + 10 | 1 | ################################################## + 2 | 1 | ################################################## + 1.5 | 1 | ################################################## + 1 | 1 | ################################################## +'''.strip() + +chartmixed_rev_byname = ''' + 1 | 1 | ################################################## + 1.5 | 1 | ################################################## + 2 | 1 | ################################################## + 10 | 1 | ################################################## +0x20 | 1 | ################################################## +0xzz | 1 | ################################################## +None | 1 | ################################################## + abc | 1 | ################################################## + nan | 1 | ################################################## +'''.strip() + class StatsTest(s_test.SynTest): @@ -230,6 +268,34 @@ async def test_stormlib_stats_countby(self): with self.raises(s_exc.BadArg): self.len(15, await core.nodes('inet:ipv4 | stats.countby ({})')) + async def test_stormlib_stats_countby_byname_mixed(self): + + async with self.getTestCore() as core: + + # a tally of numeric values where some nodes do not have the property + # set tallies the unset nodes under None. ( SYN-9957 ) + q = '''[ (inet:ipv4=0 :asn=0) (inet:ipv4=1 :asn=1) + (inet:ipv4=2 :asn=1) (inet:ipv4=3) (inet:ipv4=4 :asn=4) ]''' + self.len(5, await core.nodes(q)) + + msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --by-name') + self.stormIsInPrint(chartunset_byname, msgs) + + msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --by-name --reverse') + self.stormIsInPrint(chartunset_rev_byname, msgs) + + # numeric names sort numerically and non-numeric names sort + # lexicographically after them. + q = '''[ it:dev:str="1" it:dev:str="1.5" it:dev:str="2" it:dev:str="10" it:dev:str="nan" + it:dev:str="0x20" it:dev:str="0xzz" it:dev:str="None" it:dev:str="abc" ]''' + self.len(9, await core.nodes(q)) + + msgs = await core.stormlist('it:dev:str | stats.countby --by-name') + self.stormIsInPrint(chartmixed_byname, msgs) + + msgs = await core.stormlist('it:dev:str | stats.countby --by-name --reverse') + self.stormIsInPrint(chartmixed_rev_byname, msgs) + async def test_stormlib_stats_tally(self): async with self.getTestCore() as core: From fba0a5d733e71f483c466d9a188fac5b8a80ff57 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 21 Sep 2026 16:12:45 +0000 Subject: [PATCH 2/2] Fold the SYN-9957 regression tests into the existing test Cortex The new assertions created a second getTestCore() rather than reusing the one already stood up by test_stormlib_stats_countby. Append them to the existing context instead to avoid the extra Cortex boot. The added inet:ipv4 nodes are tagged and lifted by tag so the existing chart expectations are unaffected. Claude-Session: https://claude.ai/code/session_013pyj3S8NddfycqDWj9uBUS --- synapse/tests/test_lib_stormlib_stats.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/synapse/tests/test_lib_stormlib_stats.py b/synapse/tests/test_lib_stormlib_stats.py index 3684f92f63..f0729bd8ad 100644 --- a/synapse/tests/test_lib_stormlib_stats.py +++ b/synapse/tests/test_lib_stormlib_stats.py @@ -268,20 +268,16 @@ async def test_stormlib_stats_countby(self): with self.raises(s_exc.BadArg): self.len(15, await core.nodes('inet:ipv4 | stats.countby ({})')) - async def test_stormlib_stats_countby_byname_mixed(self): - - async with self.getTestCore() as core: - # a tally of numeric values where some nodes do not have the property # set tallies the unset nodes under None. ( SYN-9957 ) - q = '''[ (inet:ipv4=0 :asn=0) (inet:ipv4=1 :asn=1) - (inet:ipv4=2 :asn=1) (inet:ipv4=3) (inet:ipv4=4 :asn=4) ]''' + q = '''[ (inet:ipv4=1.2.3.1 :asn=0) (inet:ipv4=1.2.3.2 :asn=1) (inet:ipv4=1.2.3.3 :asn=1) + (inet:ipv4=1.2.3.4) (inet:ipv4=1.2.3.5 :asn=4) +#unset ]''' self.len(5, await core.nodes(q)) - msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --by-name') + msgs = await core.stormlist('inet:ipv4#unset | stats.countby :asn --by-name') self.stormIsInPrint(chartunset_byname, msgs) - msgs = await core.stormlist('inet:ipv4 | stats.countby :asn --by-name --reverse') + msgs = await core.stormlist('inet:ipv4#unset | stats.countby :asn --by-name --reverse') self.stormIsInPrint(chartunset_rev_byname, msgs) # numeric names sort numerically and non-numeric names sort