From 37903a807043f4b6e5d074d9235c198b8ab6b600 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 13:00:57 +0000 Subject: [PATCH 01/14] SYN-11748: Avoid KeyError reading the Axon version from axoninfo Cortex.axoninfo is an empty dict until the Axon connects, so indexing axoninfo['synapse']['version'] raised a bare KeyError. The known path is $lib.inet.http.post() with sha256 fields, which never waits on axready. Add stormtypes.getAxonVersion(), which reads the version with get() and raises FeatureNotSupported if it is missing, and use it at every site that previously indexed into axoninfo. --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 8 +++++ synapse/lib/stormhttp.py | 2 +- synapse/lib/stormtypes.py | 25 ++++++++++++-- synapse/tests/test_lib_stormhttp.py | 34 +++++++++++++++++++ synapse/tests/test_lib_stormtypes.py | 32 +++++++++++++++++ 5 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 changes/c25ec6ffe22011d527e15cec09f6f6be.yaml diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml new file mode 100644 index 00000000000..f6f0ada4997 --- /dev/null +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -0,0 +1,8 @@ +--- +desc: Fixed a KeyError in Storm Axon APIs, and in $lib.inet.http requests which upload + files from the Axon, when the Axon version was unavailable. A FeatureNotSupported + exception is now raised. +desc:literal: false +prs: [] +type: bug +... diff --git a/synapse/lib/stormhttp.py b/synapse/lib/stormhttp.py index 40610f6240a..cae06351b5e 100644 --- a/synapse/lib/stormhttp.py +++ b/synapse/lib/stormhttp.py @@ -425,7 +425,7 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, kwargs['proxy'] = proxy if ssl_opts is not None: - axonvers = self.runt.snap.core.axoninfo['synapse']['version'] + axonvers = await s_stormtypes.getAxonVersion(self.runt) mesg = f'The ssl_opts argument requires an Axon Synapse version {s_stormtypes.AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, s_stormtypes.AXON_MINVERS_SSLOPTS, mesg=mesg) diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index a5f1751d8a0..3e276653be1 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -108,6 +108,25 @@ async def resolveCoreProxyUrl(valu): case _: raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.') +async def getAxonVersion(runt): + ''' + Get the Synapse version of the Cortex's Axon. + + Args: + runt (Runtime): The Storm runtime. + + Returns: + tuple: The Synapse version of the Axon. + + Raises: + s_exc.FeatureNotSupported: If the Axon version is unavailable. + ''' + if (axonvers := runt.snap.core.axoninfo.get('synapse', {}).get('version')) is None: + mesg = 'Unable to determine the Synapse version of the Axon.' + raise s_exc.FeatureNotSupported(mesg=mesg) + + return axonvers + async def resolveAxonProxyArg(valu): ''' Resolve a proxy value to the kwarg to set for an Axon HTTP call. @@ -120,7 +139,7 @@ async def resolveAxonProxyArg(valu): ''' runt = s_scope.get('runt') - axonvers = runt.snap.core.axoninfo['synapse']['version'] + axonvers = await getAxonVersion(runt) if axonvers < AXON_MINVERS_PROXY: await runt.snap.warnonce(f'Axon version does not support proxy argument: {axonvers} < {AXON_MINVERS_PROXY}') return False, None @@ -2556,7 +2575,7 @@ async def wget(self, url, headers=None, params=None, method='GET', json=None, bo kwargs['proxy'] = proxy if ssl_opts is not None: - axonvers = self.runt.snap.core.axoninfo['synapse']['version'] + axonvers = await getAxonVersion(self.runt) mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) @@ -2597,7 +2616,7 @@ async def wput(self, sha256, url, headers=None, params=None, method='PUT', kwargs['proxy'] = proxy if ssl_opts is not None: - axonvers = self.runt.snap.core.axoninfo['synapse']['version'] + axonvers = await getAxonVersion(self.runt) mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) diff --git a/synapse/tests/test_lib_stormhttp.py b/synapse/tests/test_lib_stormhttp.py index 3f8f16deea0..80d9d3ed318 100644 --- a/synapse/tests/test_lib_stormhttp.py +++ b/synapse/tests/test_lib_stormhttp.py @@ -662,6 +662,40 @@ async def test_storm_http_post_file(self): self.eq(code, -1) self.eq('ValueError', errname) + async def test_storm_http_post_file_axon_info_missing(self): + + async with self.getTestCore() as core: + + size, sha256 = await core.axon.put(b'vertex') + opts = {'vars': {'sha256': s_common.ehex(sha256), 'url': 'http://127.0.0.1:1/'}} + + queries = ( + ''' + $fields = ([{"name": "file", "sha256": $sha256}]) + return($lib.inet.http.post($url, fields=$fields)) + ''', + ''' + $fields = ([{"name": "file", "sha256": $sha256}]) + return($lib.inet.http.post($url, fields=$fields, ssl_opts=({"verify": (false)}))) + ''', + ) + + originfo = core.axoninfo + try: + for axoninfo in ({}, {'synapse': {}}): + core.axoninfo = axoninfo + for query in queries: + with self.raises(s_exc.FeatureNotSupported) as cm: + await core.callStorm(query, opts=opts) + self.eq(cm.exception.get('mesg'), 'Unable to determine the Synapse version of the Axon.') + + finally: + core.axoninfo = originfo + + for query in queries: + resp = await core.callStorm(query, opts=opts) + self.eq(resp.get('code'), -1) + async def test_storm_http_proxy(self): conf = {'http:proxy': 'socks5://user:pass@127.0.0.1:1'} async with self.getTestCore(conf=conf) as core: diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index 1c57b8b04de..a6059f465ac 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -8199,6 +8199,38 @@ async def fake(): with self.raises(s_exc.BadState): await core.callStorm(merging) + async def test_storm_lib_axon_info_missing(self): + + async with self.getTestCore() as core: + + size, sha256 = await core.axon.put(b'vertex') + opts = {'vars': {'sha256': s_common.ehex(sha256), 'url': 'http://127.0.0.1:1/'}} + + queries = ( + 'return($lib.axon.wget($url))', + 'return($lib.axon.wget($url, proxy=(false)))', + 'return($lib.axon.wget($url, ssl_opts=({"verify": (false)})))', + 'return($lib.axon.wput($sha256, $url))', + 'return($lib.axon.wput($sha256, $url, ssl_opts=({"verify": (false)})))', + 'yield $lib.axon.urlfile($url)', + ) + + originfo = core.axoninfo + try: + for axoninfo in ({}, {'synapse': {}}): + core.axoninfo = axoninfo + for query in queries: + with self.raises(s_exc.FeatureNotSupported) as cm: + await core.callStorm(query, opts=opts) + self.eq(cm.exception.get('mesg'), 'Unable to determine the Synapse version of the Axon.') + + finally: + core.axoninfo = originfo + + for query in queries[:-1]: + resp = await core.callStorm(query, opts=opts) + self.false(resp.get('ok')) + async def test_storm_lib_axon_read_unpack(self): async with self.getTestCore() as core: From 593b1629604319b44158af253d54325b35e569fe Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 13:09:52 +0000 Subject: [PATCH 02/14] SYN-11748: Wait up to 60 seconds for the Axon before reading axoninfo Add Cortex.waitAxonReady(), which waits up to s_const.AXON_READY_TIMEOUT seconds for axready and raises TimeOut if the Axon is still not ready. getAxonVersion() and $lib.axon.unpack() now call it before reading axoninfo, so $lib.inet.http requests with sha256 fields no longer read axoninfo before the Axon connects. $lib.axon.wget() and wput() used to wait on getAxon() with no timeout; they now use waitAxonReady() and time out instead of blocking indefinitely. Update the changelog fragment to describe the timeout, and restore the double back-ticks the shell stripped from it in the previous commit. --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 8 +-- synapse/cortex.py | 15 ++++++ synapse/lib/const.py | 3 ++ synapse/lib/stormtypes.py | 10 ++-- synapse/tests/test_cortex.py | 54 +++++++++++++++++++ 5 files changed, 84 insertions(+), 6 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index f6f0ada4997..789b55b69de 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -1,7 +1,9 @@ --- -desc: Fixed a KeyError in Storm Axon APIs, and in $lib.inet.http requests which upload - files from the Axon, when the Axon version was unavailable. A FeatureNotSupported - exception is now raised. +desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and + ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from + the Axon, when the Axon was not connected. These APIs and ``$lib.axon.unpack()`` now wait + up to 60 seconds for the Axon to be ready before raising a ``TimeOut`` error, and raise a + ``FeatureNotSupported`` error if the Axon version is unavailable. desc:literal: false prs: [] type: bug diff --git a/synapse/cortex.py b/synapse/cortex.py index 7bc9185842d..40a3ba0b786 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -6172,6 +6172,21 @@ async def getAxon(self): await self.axready.wait() return self.axon.iden + async def waitAxonReady(self): + ''' + Wait for the Axon to be ready. + + Raises: + s_exc.TimeOut: If the Axon is not ready within AXON_READY_TIMEOUT seconds. + ''' + if self.axready.is_set(): + return + + timeout = s_const.AXON_READY_TIMEOUT + if not await s_coro.event_wait(self.axready, timeout=timeout): + mesg = f'Timed out waiting {timeout} seconds for the Axon to be ready.' + raise s_exc.TimeOut(mesg=mesg, timeout=timeout) + def setFeedFunc(self, name, func): ''' Set a data ingest function. diff --git a/synapse/lib/const.py b/synapse/lib/const.py index f049ccf9850..69bb4724313 100644 --- a/synapse/lib/const.py +++ b/synapse/lib/const.py @@ -52,5 +52,8 @@ MAX_LINE_SIZE = kibibyte * 64 MAX_FIELD_SIZE = kibibyte * 64 +# Axon constants +AXON_READY_TIMEOUT = 60 # seconds + # Socket constants UNIX_SOCKET_PATH_MAX = 103 diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 3e276653be1..98669df8095 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -120,7 +120,10 @@ async def getAxonVersion(runt): Raises: s_exc.FeatureNotSupported: If the Axon version is unavailable. + s_exc.TimeOut: If the Axon is not ready within AXON_READY_TIMEOUT seconds. ''' + await runt.snap.core.waitAxonReady() + if (axonvers := runt.snap.core.axoninfo.get('synapse', {}).get('version')) is None: mesg = 'Unable to determine the Synapse version of the Axon.' raise s_exc.FeatureNotSupported(mesg=mesg) @@ -2566,7 +2569,7 @@ async def wget(self, url, headers=None, params=None, method='GET', json=None, bo params = strifyHttpArg(params, multi=True) headers = strifyHttpArg(headers) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() kwargs = {} @@ -2607,7 +2610,7 @@ async def wput(self, sha256, url, headers=None, params=None, method='PUT', params = strifyHttpArg(params, multi=True) headers = strifyHttpArg(headers) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() kwargs = {} @@ -2811,6 +2814,8 @@ async def unpack(self, sha256, fmt, offs=0): ''' Unpack bytes from a file in the Axon using struct. ''' + await self.runt.snap.core.waitAxonReady() + if self.runt.snap.core.axoninfo.get('features', {}).get('unpack', 0) < 1: mesg = 'The connected Axon does not support the the unpack API. Please update your Axon.' raise s_exc.FeatureNotSupported(mesg=mesg) @@ -2822,7 +2827,6 @@ async def unpack(self, sha256, fmt, offs=0): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.unpack(s_common.uhex(sha256), fmt, offs) @registry.registerLib diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index 74083c7f039..ea4ddb273e2 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -6900,6 +6900,60 @@ async def test_cortex_axon(self): self.eq(await axon.metrics(), await core.axon.metrics()) + async def test_cortex_axon_ready_timeout(self): + + with self.getTestDir() as dirn: + + async with self.getTestAxon(dirn=dirn) as axon: + aurl = axon.getLocalUrl() + + async with self.getTestCore(conf={'axon': aurl}) as core: + + self.false(core.axready.is_set()) + self.eq(core.axoninfo, {}) + + sha256 = s_common.ehex(hashlib.sha256(b'vertex').digest()) + opts = {'vars': {'sha256': sha256, 'url': 'http://127.0.0.1:1/'}} + queries = ( + 'return($lib.axon.wget($url))', + 'return($lib.axon.wput($sha256, $url))', + 'return($lib.axon.unpack($sha256, fmt=">Q"))', + 'yield $lib.axon.urlfile($url)', + ''' + $fields = ([{"name": "file", "sha256": $sha256}]) + return($lib.inet.http.post($url, fields=$fields)) + ''', + ) + + with patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): + + with self.raises(s_exc.TimeOut) as cm: + await core.waitAxonReady() + self.eq(cm.exception.get('mesg'), 'Timed out waiting 0.1 seconds for the Axon to be ready.') + self.eq(cm.exception.get('timeout'), 0.1) + + for query in queries: + with self.raises(s_exc.TimeOut): + await core.callStorm(query, opts=opts) + + async with self.getTestAxon(dirn=dirn) as axon: + + self.true(await s_coro.event_wait(core.axready, timeout=10)) + self.nn(core.axoninfo['synapse']['version']) + + await core.axon.put(b'vertex') + + self.none(await core.waitAxonReady()) + + resp = await core.callStorm(queries[0], opts=opts) + self.false(resp.get('ok')) + + resp = await core.callStorm(queries[1], opts=opts) + self.false(resp.get('ok')) + + resp = await core.callStorm(queries[4], opts=opts) + self.eq(resp.get('code'), -1) + async def test_cortex_delLayerView(self): with self.getTestDir() as dirn: From 316f70c089aee638c1f274a2d2ca9bdf1f8ba91a Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 13:28:53 +0000 Subject: [PATCH 03/14] SYN-11748: Bound all $lib.axon waits and check unpack perms first - Use waitAxonReady() in every remaining LibAxon method, so the whole library times out consistently instead of blocking indefinitely. - Check the axon.get permission in $lib.axon.unpack() before waiting on the Axon, matching the other LibAxon methods. - Avoid building a throwaway dict default in getAxonVersion(). --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 7 ++-- synapse/lib/stormtypes.py | 32 +++++++++---------- synapse/tests/test_cortex.py | 21 ++++++++++++ 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index 789b55b69de..4c629fe7a20 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -1,9 +1,10 @@ --- desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from - the Axon, when the Axon was not connected. These APIs and ``$lib.axon.unpack()`` now wait - up to 60 seconds for the Axon to be ready before raising a ``TimeOut`` error, and raise a - ``FeatureNotSupported`` error if the Axon version is unavailable. + the Axon, when the Axon was not connected. The ``$lib.axon`` APIs and ``$lib.inet.http`` + file uploads now wait up to 60 seconds for the Axon to be ready before raising a + ``TimeOut`` error, and raise a ``FeatureNotSupported`` error if the Axon version is + unavailable. desc:literal: false prs: [] type: bug diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 98669df8095..3c6076fd775 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -124,7 +124,7 @@ async def getAxonVersion(runt): ''' await runt.snap.core.waitAxonReady() - if (axonvers := runt.snap.core.axoninfo.get('synapse', {}).get('version')) is None: + if (syninfo := runt.snap.core.axoninfo.get('synapse')) is None or (axonvers := syninfo.get('version')) is None: mesg = 'Unable to determine the Synapse version of the Axon.' raise s_exc.FeatureNotSupported(mesg=mesg) @@ -2502,7 +2502,7 @@ def getObjLocals(self): async def readlines(self, sha256, errors='ignore'): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() sha256 = await tostr(sha256) async for line in self.runt.snap.core.axon.readlines(sha256, errors=errors): @@ -2512,7 +2512,7 @@ async def readlines(self, sha256, errors='ignore'): async def jsonlines(self, sha256, errors='ignore'): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() sha256 = await tostr(sha256) async for line in self.runt.snap.core.axon.jsonlines(sha256): @@ -2530,7 +2530,7 @@ async def dels(self, sha256s): hashes = [s_common.uhex(s) for s in sha256s] - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() axon = self.runt.snap.core.axon return await axon.dels(hashes) @@ -2543,7 +2543,7 @@ async def del_(self, sha256): sha256 = await tostr(sha256) sha256b = s_common.uhex(sha256) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() axon = self.runt.snap.core.axon return await axon.del_(sha256b) @@ -2702,7 +2702,7 @@ async def list(self, offs=0, wait=False, timeout=None): if not self.runt.allowed(('axon', 'has')): self.runt.confirm(('storm', 'lib', 'axon', 'has')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() axon = self.runt.snap.core.axon async for item in axon.hashes(offs, wait=wait, timeout=timeout): @@ -2714,7 +2714,7 @@ async def csvrows(self, sha256, dialect='excel', errors='ignore', **fmtparams): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() sha256 = await tostr(sha256) dialect = await tostr(dialect) @@ -2734,7 +2734,7 @@ async def upload(self, genr): self.runt.confirm(('axon', 'upload')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() async with await self.runt.snap.core.axon.upload() as upload: async for byts in s_coro.agen(genr): await upload.write(byts) @@ -2749,7 +2749,7 @@ async def has(self, sha256): self.runt.confirm(('axon', 'has')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() return await self.runt.snap.core.axon.has(s_common.uhex(sha256)) @stormfunc(readonly=True) @@ -2758,7 +2758,7 @@ async def size(self, sha256): self.runt.confirm(('axon', 'has')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() return await self.runt.snap.core.axon.size(s_common.uhex(sha256)) async def put(self, byts): @@ -2770,7 +2770,7 @@ async def put(self, byts): sha256 = hashlib.sha256(byts).digest() - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() if await self.runt.snap.core.axon.has(sha256): return (len(byts), s_common.ehex(sha256)) @@ -2783,7 +2783,7 @@ async def hashset(self, sha256): self.runt.confirm(('axon', 'has')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() return await self.runt.snap.core.axon.hashset(s_common.uhex(sha256)) @stormfunc(readonly=True) @@ -2802,7 +2802,7 @@ async def read(self, sha256, offs=0, size=s_const.mebibyte): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() byts = b'' async for chunk in self.runt.snap.core.axon.get(s_common.uhex(sha256), offs=offs, size=size): @@ -2814,6 +2814,9 @@ async def unpack(self, sha256, fmt, offs=0): ''' Unpack bytes from a file in the Axon using struct. ''' + if not self.runt.allowed(('axon', 'get')): + self.runt.confirm(('storm', 'lib', 'axon', 'get')) + await self.runt.snap.core.waitAxonReady() if self.runt.snap.core.axoninfo.get('features', {}).get('unpack', 0) < 1: @@ -2824,9 +2827,6 @@ async def unpack(self, sha256, fmt, offs=0): fmt = await tostr(fmt) offs = await toint(offs) - if not self.runt.allowed(('axon', 'get')): - self.runt.confirm(('storm', 'lib', 'axon', 'get')) - return await self.runt.snap.core.axon.unpack(s_common.uhex(sha256), fmt, offs) @registry.registerLib diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index ea4ddb273e2..5e86f3977bc 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -6923,7 +6923,23 @@ async def test_cortex_axon_ready_timeout(self): $fields = ([{"name": "file", "sha256": $sha256}]) return($lib.inet.http.post($url, fields=$fields)) ''', + 'for $line in $lib.axon.readlines($sha256) {}', + 'for $item in $lib.axon.jsonlines($sha256) {}', + 'for $row in $lib.axon.csvrows($sha256) {}', + 'for $item in $lib.axon.list() {}', + 'return($lib.axon.dels(($sha256,)))', + 'return($lib.axon.del($sha256))', + 'return($lib.axon.upload(([])))', + 'return($lib.axon.has($sha256))', + 'return($lib.axon.size($sha256))', + 'return($lib.axon.put($buf))', + 'return($lib.axon.hashset($sha256))', + 'return($lib.axon.read($sha256))', ) + opts['vars']['buf'] = b'vertex' + + visi = await core.auth.addUser('visi') + visiopts = {'user': visi.iden, 'vars': opts['vars']} with patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): @@ -6936,6 +6952,11 @@ async def test_cortex_axon_ready_timeout(self): with self.raises(s_exc.TimeOut): await core.callStorm(query, opts=opts) + # permission checks run before waiting on the Axon + for query in queries: + with self.raises(s_exc.AuthDeny): + await core.callStorm(query, opts=visiopts) + async with self.getTestAxon(dirn=dirn) as axon: self.true(await s_coro.event_wait(core.axready, timeout=10)) From 44556cbd7dd824cc08d46704d8ea7d4ccc3aafb9 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 13:32:40 +0000 Subject: [PATCH 04/14] SYN-11748: Report Axon readiness and version in Cortex getCellInfo() Add axon:ready and axon:version to the cell section of the Cortex getCellInfo() output. axon:version is the Synapse version most recently reported by the Axon, or None before the Cortex first connects to it. --- changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml | 8 +++++++ docs/synapse/devopsguide.rst | 4 ++++ synapse/cortex.py | 23 +++++++++++++++++++ synapse/tests/test_cortex.py | 22 ++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml diff --git a/changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml b/changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml new file mode 100644 index 00000000000..20f5bf7f433 --- /dev/null +++ b/changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml @@ -0,0 +1,8 @@ +--- +desc: Added ``axon:ready`` and ``axon:version`` keys to the ``cell`` section of the + Cortex ``getCellInfo()`` API, which report whether the Axon is ready and the Synapse + version most recently reported by the Axon. +desc:literal: false +prs: [] +type: feat +... diff --git a/docs/synapse/devopsguide.rst b/docs/synapse/devopsguide.rst index d36fef71b9b..d3ceebeb7e6 100644 --- a/docs/synapse/devopsguide.rst +++ b/docs/synapse/devopsguide.rst @@ -1340,6 +1340,10 @@ Docker Image: ``vertexproject/synapse-axon:v2.x.x`` it is **highly** recommended that you install it as a separated service to help distribute load and allow direct access by other Advanced Power-Ups. +The Cortex reports the status of its Axon in the ``cell`` section of the dictionary returned by ``getCellInfo()``. +The ``axon:ready`` key is ``true`` when the Axon is ready, and the ``axon:version`` key contains the Synapse version +most recently reported by the Axon, or ``null`` if the Cortex has not connected to it. + **Configuration** A typical Axon deployment does not require any additional configuration. For the full list supported options, see the diff --git a/synapse/cortex.py b/synapse/cortex.py index 40a3ba0b786..1c3f6697396 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -6187,6 +6187,29 @@ async def waitAxonReady(self): mesg = f'Timed out waiting {timeout} seconds for the Axon to be ready.' raise s_exc.TimeOut(mesg=mesg, timeout=timeout) + async def getCellInfo(self): + ''' + Return metadata specific for the Cortex. + + Notes: + In addition to the base Cell information, the ``cell`` section + includes ``axon:ready``, which is True when the Axon is ready, and + ``axon:version``, the Synapse version most recently reported by + the Axon or None if the Cortex has not connected to it. + + Returns: + Dict: A Dictionary of metadata. + ''' + info = await super().getCellInfo() + + axonvers = None + if (syninfo := self.axoninfo.get('synapse')) is not None: + axonvers = syninfo.get('version') + + info['cell']['axon:ready'] = self.axready.is_set() + info['cell']['axon:version'] = axonvers + return info + def setFeedFunc(self, name, func): ''' Set a data ingest function. diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index 5e86f3977bc..67865a33763 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -6863,6 +6863,10 @@ async def test_cortex_axon(self): self.eq(size, 8) self.eq(s_common.ehex(sha2), '2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892') self.true(core.nexsroot is core.axon.nexsroot) + + info = await core.getCellInfo() + self.true(info['cell']['axon:ready']) + self.eq(info['cell']['axon:version'], s_version.version) self.true(core.axon.isfini) self.false(core.axready.is_set()) @@ -6912,6 +6916,10 @@ async def test_cortex_axon_ready_timeout(self): self.false(core.axready.is_set()) self.eq(core.axoninfo, {}) + info = await core.getCellInfo() + self.false(info['cell']['axon:ready']) + self.none(info['cell']['axon:version']) + sha256 = s_common.ehex(hashlib.sha256(b'vertex').digest()) opts = {'vars': {'sha256': sha256, 'url': 'http://127.0.0.1:1/'}} queries = ( @@ -6962,6 +6970,10 @@ async def test_cortex_axon_ready_timeout(self): self.true(await s_coro.event_wait(core.axready, timeout=10)) self.nn(core.axoninfo['synapse']['version']) + info = await core.callStorm('return($lib.cell.getCellInfo())') + self.true(info['cell']['axon:ready']) + self.eq(info['cell']['axon:version'], s_version.version) + await core.axon.put(b'vertex') self.none(await core.waitAxonReady()) @@ -6975,6 +6987,16 @@ async def test_cortex_axon_ready_timeout(self): resp = await core.callStorm(queries[4], opts=opts) self.eq(resp.get('code'), -1) + # the last version reported by the Axon remains after it disconnects + for _ in range(20): + if not core.axready.is_set(): + break + await asyncio.sleep(0.1) + + info = await core.getCellInfo() + self.false(info['cell']['axon:ready']) + self.eq(info['cell']['axon:version'], s_version.version) + async def test_cortex_delLayerView(self): with self.getTestDir() as dirn: From e115619eb3406b19317a4d2df8ba5bb0f4ad9253 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 13:33:15 +0000 Subject: [PATCH 05/14] SYN-11748: Raise AXON_READY_TIMEOUT to 300 seconds Match the default timeout of the $lib.inet.http request APIs. --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 2 +- synapse/lib/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index 4c629fe7a20..c7db9ec1839 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -2,7 +2,7 @@ desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from the Axon, when the Axon was not connected. The ``$lib.axon`` APIs and ``$lib.inet.http`` - file uploads now wait up to 60 seconds for the Axon to be ready before raising a + file uploads now wait up to 300 seconds for the Axon to be ready before raising a ``TimeOut`` error, and raise a ``FeatureNotSupported`` error if the Axon version is unavailable. desc:literal: false diff --git a/synapse/lib/const.py b/synapse/lib/const.py index 69bb4724313..71414f41c39 100644 --- a/synapse/lib/const.py +++ b/synapse/lib/const.py @@ -53,7 +53,7 @@ MAX_FIELD_SIZE = kibibyte * 64 # Axon constants -AXON_READY_TIMEOUT = 60 # seconds +AXON_READY_TIMEOUT = 300 # seconds # Socket constants UNIX_SOCKET_PATH_MAX = 103 From c6d70336b63e8cd5d42d5637e5620227918e6d73 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 14:42:40 +0000 Subject: [PATCH 06/14] SYN-11748: Bound the $lib.bytes Axon waits Use waitAxonReady() in the deprecated LibBytes methods, matching LibAxon. --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 4 ++-- synapse/lib/stormtypes.py | 10 +++++----- synapse/tests/test_cortex.py | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index c7db9ec1839..e097340b138 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -1,8 +1,8 @@ --- desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from - the Axon, when the Axon was not connected. The ``$lib.axon`` APIs and ``$lib.inet.http`` - file uploads now wait up to 300 seconds for the Axon to be ready before raising a + the Axon, when the Axon was not connected. The ``$lib.axon`` and ``$lib.bytes`` APIs and + ``$lib.inet.http`` file uploads now wait up to 300 seconds for the Axon to be ready before raising a ``TimeOut`` error, and raise a ``FeatureNotSupported`` error if the Axon version is unavailable. desc:literal: false diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 3c6076fd775..8a291c26052 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -2964,7 +2964,7 @@ async def _libBytesUpload(self, genr): self.runt.confirm(('axon', 'upload'), default=True) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() async with await self.runt.snap.core.axon.upload() as upload: async for byts in s_coro.agen(genr): await upload.write(byts) @@ -2991,7 +2991,7 @@ async def _libBytesHas(self, sha256): self.runt.confirm(('axon', 'has'), default=True) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() todo = s_common.todo('has', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret @@ -3003,7 +3003,7 @@ async def _libBytesSize(self, sha256): self.runt.confirm(('axon', 'has'), default=True) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() todo = s_common.todo('size', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret @@ -3016,7 +3016,7 @@ async def _libBytesPut(self, byts): self.runt.confirm(('axon', 'upload'), default=True) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() todo = s_common.todo('put', byts) size, sha2 = await self.dyncall('axon', todo) @@ -3029,7 +3029,7 @@ async def _libBytesHashset(self, sha256): self.runt.confirm(('axon', 'has'), default=True) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() todo = s_common.todo('hashset', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index 67865a33763..d7a520bdab6 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -6943,10 +6943,17 @@ async def test_cortex_axon_ready_timeout(self): 'return($lib.axon.put($buf))', 'return($lib.axon.hashset($sha256))', 'return($lib.axon.read($sha256))', + 'return($lib.bytes.put($buf))', + 'return($lib.bytes.has($sha256))', + 'return($lib.bytes.size($sha256))', + 'return($lib.bytes.hashset($sha256))', + 'return($lib.bytes.upload(([])))', ) opts['vars']['buf'] = b'vertex' + # $lib.bytes permissions are allowed by default, so deny them explicitly visi = await core.auth.addUser('visi') + await visi.addRule((False, ('axon',))) visiopts = {'user': visi.iden, 'vars': opts['vars']} with patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): From cc7b975f04e1a2b2a01e560829af745074aeb8ce Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 14:49:08 +0000 Subject: [PATCH 07/14] SYN-11748: Bound the IMAP fetch Axon wait Use waitAxonReady() in the IMAP server fetch() method. --- synapse/lib/stormlib/imap.py | 2 +- synapse/tests/test_lib_stormlib_imap.py | 26 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/synapse/lib/stormlib/imap.py b/synapse/lib/stormlib/imap.py index b7e3ea00d5a..fd440d9d747 100644 --- a/synapse/lib/stormlib/imap.py +++ b/synapse/lib/stormlib/imap.py @@ -786,7 +786,7 @@ async def fetch(self, uid): # to prevent retrieving a very large blob of data. uid = await s_stormtypes.toint(uid) - await self.runt.snap.core.getAxon() + await self.runt.snap.core.waitAxonReady() axon = self.runt.snap.core.axon coro = self.imap_cli.uid_fetch(str(uid), '(RFC822)') diff --git a/synapse/tests/test_lib_stormlib_imap.py b/synapse/tests/test_lib_stormlib_imap.py index 6160a9b01ef..02542c96f83 100644 --- a/synapse/tests/test_lib_stormlib_imap.py +++ b/synapse/tests/test_lib_stormlib_imap.py @@ -883,6 +883,32 @@ async def test_storm_imap_fetch(self): self.eq(ret, (True, (rfc822, header, b'(UID 1 RFC822 BODY[HEADER])'))) + async def test_storm_imap_fetch_axon_timeout(self): + + async with self.getTestCoreAndImapPort() as (core, port): + user = 'user00@vertex.link' + opts = {'vars': {'port': port, 'user': user}} + + scmd = ''' + $server = $lib.inet.imap.connect(127.0.0.1, port=$port, ssl=(false)) + $server.login($user, "pass00") + $server.select("INBOX") + yield $server.fetch("1") + ''' + + core.axready.clear() + try: + with mock.patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): + with self.raises(s_exc.TimeOut): + await core.nodes(scmd, opts=opts) + + finally: + core.axready.set() + + nodes = await core.nodes(scmd, opts=opts) + self.len(1, nodes) + self.eq('file:bytes', nodes[0].ndef[0]) + async def test_storm_imap_logout(self): async with self.getTestCoreAndImapPort() as (core, port): From feb8d1e92eb820c119ba1931af85d0f549a05906 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 14:49:21 +0000 Subject: [PATCH 08/14] SYN-11748: Mention IMAP fetches in the changelog entry --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index e097340b138..17ee8ff908f 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -1,10 +1,10 @@ --- desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from - the Axon, when the Axon was not connected. The ``$lib.axon`` and ``$lib.bytes`` APIs and - ``$lib.inet.http`` file uploads now wait up to 300 seconds for the Axon to be ready before raising a - ``TimeOut`` error, and raise a ``FeatureNotSupported`` error if the Axon version is - unavailable. + the Axon, when the Axon was not connected. Storm APIs which use the Axon, including + ``$lib.axon``, ``$lib.bytes``, ``$lib.inet.http`` file uploads, and IMAP message fetches, + now wait up to 300 seconds for the Axon to be ready before raising a ``TimeOut`` error, + and raise a ``FeatureNotSupported`` error if the Axon version is unavailable. desc:literal: false prs: [] type: bug From 0f056b34ac4a8484309edaad43696f59595015b6 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 14:52:54 +0000 Subject: [PATCH 09/14] SYN-11748: Bound the delnode --delbytes Axon wait Use waitAxonReady() in delnode --delbytes, after its permission check. --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 3 ++- synapse/lib/storm.py | 2 +- synapse/tests/test_lib_storm.py | 18 +++++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index 17ee8ff908f..b327149db0f 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -2,7 +2,8 @@ desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from the Axon, when the Axon was not connected. Storm APIs which use the Axon, including - ``$lib.axon``, ``$lib.bytes``, ``$lib.inet.http`` file uploads, and IMAP message fetches, + ``$lib.axon``, ``$lib.bytes``, ``$lib.inet.http`` file uploads, IMAP message fetches, and + ``delnode --delbytes``, now wait up to 300 seconds for the Axon to be ready before raising a ``TimeOut`` error, and raise a ``FeatureNotSupported`` error if the Axon version is unavailable. desc:literal: false diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index 032499a0988..4bcef3e2961 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -4429,7 +4429,7 @@ async def execStormCmd(self, runt, genr): if delbytes: runt.confirm(('storm', 'lib', 'axon', 'del')) - await runt.snap.core.getAxon() + await runt.snap.core.waitAxonReady() axon = runt.snap.core.axon async for node, path in genr: diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index 2e4056ce3ff..396195f53f5 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -6289,7 +6289,23 @@ async def test_lib_storm_delnode(self): with self.raises(s_exc.AuthDeny): await asvisi.callStorm(f'file:bytes={sha256} | delnode --delbytes') - await visi.addRule((True, ('storm', 'lib', 'axon', 'del'))) + core.axready.clear() + try: + with mock.patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): + + # permission checks run before waiting on the Axon + with self.raises(s_exc.AuthDeny): + await asvisi.callStorm(f'file:bytes={sha256} | delnode --delbytes') + + await visi.addRule((True, ('storm', 'lib', 'axon', 'del'))) + + with self.raises(s_exc.TimeOut): + await asvisi.callStorm(f'file:bytes={sha256} | delnode --delbytes') + + self.len(1, await core.nodes(f'file:bytes={sha256}')) + + finally: + core.axready.set() await asvisi.callStorm(f'file:bytes={sha256} | delnode --delbytes') self.len(0, await core.nodes(f'file:bytes={sha256}')) From b4fd1aae1293175c8786140fa1cf8bb827c30919 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 15:34:01 +0000 Subject: [PATCH 10/14] SYN-11748: Bound Cortex.getAxon() and the telepath/HTTP Axon waits - Cortex.getAxon() now takes timeout=s_const.AXON_READY_TIMEOUT and raises TimeOut, with timeout=None waiting indefinitely. It returns the Axon iden from axoninfo, because self.axon.iden was a telepath Method object for a remote Axon. - Drop the unreleased waitAxonReady() and move its callers to getAxon(). - CoreApi.getAxonUpload() and getAxonBytes() use the bounded wait. - The Cortex Axon HTTP handlers return a 503 TimeOut error when the Axon is not ready, instead of hanging. --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 11 ++--- synapse/cortex.py | 43 +++++++++++------- synapse/lib/storm.py | 2 +- synapse/lib/stormlib/imap.py | 2 +- synapse/lib/stormtypes.py | 44 +++++++++---------- synapse/tests/test_cortex.py | 38 +++++++++++++--- synapse/tests/test_lib_httpapi.py | 22 ++++++++++ synapse/tests/test_lib_storm.py | 3 +- synapse/tests/test_lib_stormlib_imap.py | 3 +- 9 files changed, 115 insertions(+), 53 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index b327149db0f..3dee80da63a 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -1,11 +1,12 @@ --- desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from - the Axon, when the Axon was not connected. Storm APIs which use the Axon, including - ``$lib.axon``, ``$lib.bytes``, ``$lib.inet.http`` file uploads, IMAP message fetches, and - ``delnode --delbytes``, - now wait up to 300 seconds for the Axon to be ready before raising a ``TimeOut`` error, - and raise a ``FeatureNotSupported`` error if the Axon version is unavailable. + the Axon, when the Axon was not connected. Cortex Storm, Telepath, and HTTP APIs which + use the Axon now wait up to 300 seconds for the Axon to be ready before raising a + ``TimeOut`` error, and Storm APIs raise a ``FeatureNotSupported`` error if the Axon + version is unavailable. The Cortex ``getAxon()`` method now accepts a ``timeout`` + argument, which defaults to 300 seconds, and returns the Axon iden when the Axon is + remote. desc:literal: false prs: [] type: bug diff --git a/synapse/cortex.py b/synapse/cortex.py index 1c3f6697396..71e33dcf77f 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -1,5 +1,6 @@ import os import copy +import http import regex import asyncio import logging @@ -176,7 +177,13 @@ async def wrap_liftgenr(iden, genr): class CortexAxonMixin: async def prepare(self): - await self.cell.axready.wait() + try: + await self.cell.getAxon() + except s_exc.TimeOut as e: + self.sendRestExc(e, status_code=http.HTTPStatus.SERVICE_UNAVAILABLE) + await self.finish() + return + await s_coro.ornot(super().prepare) def getAxon(self): @@ -192,7 +199,8 @@ class CortexAxonHttpDelV1(CortexAxonMixin, s_axon.AxonHttpDelV1): pass class CortexAxonHttpUploadV1(CortexAxonMixin, s_axon.AxonHttpUploadV1): - pass + # set in prepare(), which is skipped if the Axon is not ready + upfd = None class CortexAxonHttpBySha256V1(CortexAxonMixin, s_axon.AxonHttpBySha256V1): pass @@ -760,13 +768,13 @@ async def iterTagPropRows(self, layriden, tag, prop, form=None, stortype=None, s async def getAxonUpload(self): self.user.confirm(('axon', 'upload')) - await self.cell.axready.wait() + await self.cell.getAxon() upload = await self.cell.axon.upload() return await s_axon.UpLoadProxy.anit(self.link, upload) async def getAxonBytes(self, sha256): self.user.confirm(('axon', 'get')) - await self.cell.axready.wait() + await self.cell.getAxon() async for byts in self.cell.axon.get(s_common.uhex(sha256)): yield byts @@ -6168,25 +6176,28 @@ def getStormLib(self, path): def getStormCmds(self): return list(self.stormcmds.items()) - async def getAxon(self): - await self.axready.wait() - return self.axon.iden - - async def waitAxonReady(self): + async def getAxon(self, timeout=s_const.AXON_READY_TIMEOUT): ''' - Wait for the Axon to be ready. + Wait for the Axon to be ready and return its iden. + + Args: + timeout (int): The maximum number of seconds to wait, or None to wait indefinitely. + + Returns: + str: The iden of the Axon, or None if the Axon did not report one. Raises: - s_exc.TimeOut: If the Axon is not ready within AXON_READY_TIMEOUT seconds. + s_exc.TimeOut: If the Axon is not ready within the timeout. ''' - if self.axready.is_set(): - return - - timeout = s_const.AXON_READY_TIMEOUT - if not await s_coro.event_wait(self.axready, timeout=timeout): + if not self.axready.is_set() and not await s_coro.event_wait(self.axready, timeout=timeout): mesg = f'Timed out waiting {timeout} seconds for the Axon to be ready.' raise s_exc.TimeOut(mesg=mesg, timeout=timeout) + if (cellinfo := self.axoninfo.get('cell')) is None: + return None + + return cellinfo.get('iden') + async def getCellInfo(self): ''' Return metadata specific for the Cortex. diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index 4bcef3e2961..032499a0988 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -4429,7 +4429,7 @@ async def execStormCmd(self, runt, genr): if delbytes: runt.confirm(('storm', 'lib', 'axon', 'del')) - await runt.snap.core.waitAxonReady() + await runt.snap.core.getAxon() axon = runt.snap.core.axon async for node, path in genr: diff --git a/synapse/lib/stormlib/imap.py b/synapse/lib/stormlib/imap.py index fd440d9d747..b7e3ea00d5a 100644 --- a/synapse/lib/stormlib/imap.py +++ b/synapse/lib/stormlib/imap.py @@ -786,7 +786,7 @@ async def fetch(self, uid): # to prevent retrieving a very large blob of data. uid = await s_stormtypes.toint(uid) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() axon = self.runt.snap.core.axon coro = self.imap_cli.uid_fetch(str(uid), '(RFC822)') diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 8a291c26052..dfb862cd1f6 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -120,9 +120,9 @@ async def getAxonVersion(runt): Raises: s_exc.FeatureNotSupported: If the Axon version is unavailable. - s_exc.TimeOut: If the Axon is not ready within AXON_READY_TIMEOUT seconds. + s_exc.TimeOut: If the Axon is not ready within the default Axon timeout. ''' - await runt.snap.core.waitAxonReady() + await runt.snap.core.getAxon() if (syninfo := runt.snap.core.axoninfo.get('synapse')) is None or (axonvers := syninfo.get('version')) is None: mesg = 'Unable to determine the Synapse version of the Axon.' @@ -2502,7 +2502,7 @@ def getObjLocals(self): async def readlines(self, sha256, errors='ignore'): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() sha256 = await tostr(sha256) async for line in self.runt.snap.core.axon.readlines(sha256, errors=errors): @@ -2512,7 +2512,7 @@ async def readlines(self, sha256, errors='ignore'): async def jsonlines(self, sha256, errors='ignore'): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() sha256 = await tostr(sha256) async for line in self.runt.snap.core.axon.jsonlines(sha256): @@ -2530,7 +2530,7 @@ async def dels(self, sha256s): hashes = [s_common.uhex(s) for s in sha256s] - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() axon = self.runt.snap.core.axon return await axon.dels(hashes) @@ -2543,7 +2543,7 @@ async def del_(self, sha256): sha256 = await tostr(sha256) sha256b = s_common.uhex(sha256) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() axon = self.runt.snap.core.axon return await axon.del_(sha256b) @@ -2569,7 +2569,7 @@ async def wget(self, url, headers=None, params=None, method='GET', json=None, bo params = strifyHttpArg(params, multi=True) headers = strifyHttpArg(headers) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() kwargs = {} @@ -2610,7 +2610,7 @@ async def wput(self, sha256, url, headers=None, params=None, method='PUT', params = strifyHttpArg(params, multi=True) headers = strifyHttpArg(headers) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() kwargs = {} @@ -2702,7 +2702,7 @@ async def list(self, offs=0, wait=False, timeout=None): if not self.runt.allowed(('axon', 'has')): self.runt.confirm(('storm', 'lib', 'axon', 'has')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() axon = self.runt.snap.core.axon async for item in axon.hashes(offs, wait=wait, timeout=timeout): @@ -2714,7 +2714,7 @@ async def csvrows(self, sha256, dialect='excel', errors='ignore', **fmtparams): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() sha256 = await tostr(sha256) dialect = await tostr(dialect) @@ -2734,7 +2734,7 @@ async def upload(self, genr): self.runt.confirm(('axon', 'upload')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() async with await self.runt.snap.core.axon.upload() as upload: async for byts in s_coro.agen(genr): await upload.write(byts) @@ -2749,7 +2749,7 @@ async def has(self, sha256): self.runt.confirm(('axon', 'has')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.has(s_common.uhex(sha256)) @stormfunc(readonly=True) @@ -2758,7 +2758,7 @@ async def size(self, sha256): self.runt.confirm(('axon', 'has')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.size(s_common.uhex(sha256)) async def put(self, byts): @@ -2770,7 +2770,7 @@ async def put(self, byts): sha256 = hashlib.sha256(byts).digest() - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() if await self.runt.snap.core.axon.has(sha256): return (len(byts), s_common.ehex(sha256)) @@ -2783,7 +2783,7 @@ async def hashset(self, sha256): self.runt.confirm(('axon', 'has')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.hashset(s_common.uhex(sha256)) @stormfunc(readonly=True) @@ -2802,7 +2802,7 @@ async def read(self, sha256, offs=0, size=s_const.mebibyte): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() byts = b'' async for chunk in self.runt.snap.core.axon.get(s_common.uhex(sha256), offs=offs, size=size): @@ -2817,7 +2817,7 @@ async def unpack(self, sha256, fmt, offs=0): if not self.runt.allowed(('axon', 'get')): self.runt.confirm(('storm', 'lib', 'axon', 'get')) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() if self.runt.snap.core.axoninfo.get('features', {}).get('unpack', 0) < 1: mesg = 'The connected Axon does not support the the unpack API. Please update your Axon.' @@ -2964,7 +2964,7 @@ async def _libBytesUpload(self, genr): self.runt.confirm(('axon', 'upload'), default=True) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() async with await self.runt.snap.core.axon.upload() as upload: async for byts in s_coro.agen(genr): await upload.write(byts) @@ -2991,7 +2991,7 @@ async def _libBytesHas(self, sha256): self.runt.confirm(('axon', 'has'), default=True) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() todo = s_common.todo('has', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret @@ -3003,7 +3003,7 @@ async def _libBytesSize(self, sha256): self.runt.confirm(('axon', 'has'), default=True) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() todo = s_common.todo('size', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret @@ -3016,7 +3016,7 @@ async def _libBytesPut(self, byts): self.runt.confirm(('axon', 'upload'), default=True) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() todo = s_common.todo('put', byts) size, sha2 = await self.dyncall('axon', todo) @@ -3029,7 +3029,7 @@ async def _libBytesHashset(self, sha256): self.runt.confirm(('axon', 'has'), default=True) - await self.runt.snap.core.waitAxonReady() + await self.runt.snap.core.getAxon() todo = s_common.todo('hashset', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) return ret diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index d7a520bdab6..05f240fe352 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -4,7 +4,9 @@ import time import asyncio import hashlib +import inspect import logging +import functools import regex @@ -20,6 +22,7 @@ import synapse.lib.coro as s_coro import synapse.lib.node as s_node import synapse.lib.time as s_time +import synapse.lib.const as s_const import synapse.lib.layer as s_layer import synapse.lib.storm as s_storm import synapse.lib.output as s_output @@ -6863,6 +6866,7 @@ async def test_cortex_axon(self): self.eq(size, 8) self.eq(s_common.ehex(sha2), '2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892') self.true(core.nexsroot is core.axon.nexsroot) + self.eq(await core.getAxon(), core.axon.iden) info = await core.getCellInfo() self.true(info['cell']['axon:ready']) @@ -6956,12 +6960,24 @@ async def test_cortex_axon_ready_timeout(self): await visi.addRule((False, ('axon',))) visiopts = {'user': visi.iden, 'vars': opts['vars']} - with patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): + timeout = inspect.signature(core.getAxon).parameters['timeout'].default + self.eq(timeout, s_const.AXON_READY_TIMEOUT) - with self.raises(s_exc.TimeOut) as cm: - await core.waitAxonReady() - self.eq(cm.exception.get('mesg'), 'Timed out waiting 0.1 seconds for the Axon to be ready.') - self.eq(cm.exception.get('timeout'), 0.1) + with self.raises(s_exc.TimeOut) as cm: + await core.getAxon(timeout=0.1) + self.eq(cm.exception.get('mesg'), 'Timed out waiting 0.1 seconds for the Axon to be ready.') + self.eq(cm.exception.get('timeout'), 0.1) + + with patch.object(core, 'getAxon', functools.partial(core.getAxon, timeout=0.1)): + + async with core.getLocalProxy() as proxy: + + with self.raises(s_exc.TimeOut): + await proxy.getAxonUpload() + + with self.raises(s_exc.TimeOut): + async for byts in proxy.getAxonBytes(sha256): + pass for query in queries: with self.raises(s_exc.TimeOut): @@ -6983,7 +6999,17 @@ async def test_cortex_axon_ready_timeout(self): await core.axon.put(b'vertex') - self.none(await core.waitAxonReady()) + self.eq(await core.getAxon(), axon.iden) + self.eq(await core.getAxon(timeout=None), axon.iden) + + originfo = core.axoninfo + try: + for axoninfo in ({}, {'cell': {}}): + core.axoninfo = axoninfo + self.none(await core.getAxon()) + + finally: + core.axoninfo = originfo resp = await core.callStorm(queries[0], opts=opts) self.false(resp.get('ok')) diff --git a/synapse/tests/test_lib_httpapi.py b/synapse/tests/test_lib_httpapi.py index 119b50c2828..0cce5fac140 100644 --- a/synapse/tests/test_lib_httpapi.py +++ b/synapse/tests/test_lib_httpapi.py @@ -1,5 +1,8 @@ import ssl import http +import functools + +from unittest import mock import aiohttp import aiohttp.client_exceptions as a_exc @@ -2202,6 +2205,25 @@ async def test_core_remote_axon_http(self): async with sess.get(url, timeout=timeout) as resp: pass + with mock.patch.object(core, 'getAxon', functools.partial(core.getAxon, timeout=0.1)): + + async with sess.get(url) as resp: + self.eq(resp.status, http.HTTPStatus.SERVICE_UNAVAILABLE) + item = await resp.json() + self.eq(item.get('status'), 'err') + self.eq(item.get('code'), 'TimeOut') + self.eq(item.get('mesg'), 'Timed out waiting 0.1 seconds for the Axon to be ready.') + + # the upload handler cleans up without a prepared upload + url = f'https://localhost:{port}/api/v1/axon/files/put' + with self.getLoggerStream('tornado.application') as stream: + async with sess.post(url, data=b'asdfasdf') as resp: + self.eq(resp.status, http.HTTPStatus.SERVICE_UNAVAILABLE) + item = await resp.json() + self.eq(item.get('code'), 'TimeOut') + + self.notin('Uncaught exception', stream.getvalue()) + async def test_http_login_broken(self): async with self.getTestCore() as core: diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index 396195f53f5..71fd0e50cd9 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -1,6 +1,7 @@ import copy import asyncio import textwrap +import functools import itertools import urllib.parse as u_parse import unittest.mock as mock @@ -6291,7 +6292,7 @@ async def test_lib_storm_delnode(self): core.axready.clear() try: - with mock.patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): + with mock.patch.object(core, 'getAxon', functools.partial(core.getAxon, timeout=0.1)): # permission checks run before waiting on the Axon with self.raises(s_exc.AuthDeny): diff --git a/synapse/tests/test_lib_stormlib_imap.py b/synapse/tests/test_lib_stormlib_imap.py index 02542c96f83..66f18b69e42 100644 --- a/synapse/tests/test_lib_stormlib_imap.py +++ b/synapse/tests/test_lib_stormlib_imap.py @@ -3,6 +3,7 @@ import imaplib import logging import textwrap +import functools import contextlib import regex @@ -898,7 +899,7 @@ async def test_storm_imap_fetch_axon_timeout(self): core.axready.clear() try: - with mock.patch('synapse.lib.const.AXON_READY_TIMEOUT', 0.1): + with mock.patch.object(core, 'getAxon', functools.partial(core.getAxon, timeout=0.1)): with self.raises(s_exc.TimeOut): await core.nodes(scmd, opts=opts) From f908a4143fa95785cd8e882d16ada0f11cea6e67 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 15:38:56 +0000 Subject: [PATCH 11/14] SYN-11748: Wait for the Axon after HTTP permission checks Move the Cortex Axon HTTP handlers' Axon wait from prepare() into an allowed() override, so an unauthenticated or unauthorized request is refused immediately instead of waiting on the Axon first. Return from AxonHttpUploadV1.prepare() after a denied request, which previously went on to start an upload in the Axon. --- changes/391b55492c8a85ae68deb9654729681e.yaml | 7 ++++ synapse/axon.py | 1 + synapse/cortex.py | 14 ++++--- synapse/tests/test_axon.py | 11 ++++-- synapse/tests/test_lib_httpapi.py | 37 +++++++++++++++---- 5 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 changes/391b55492c8a85ae68deb9654729681e.yaml diff --git a/changes/391b55492c8a85ae68deb9654729681e.yaml b/changes/391b55492c8a85ae68deb9654729681e.yaml new file mode 100644 index 00000000000..15c36a14449 --- /dev/null +++ b/changes/391b55492c8a85ae68deb9654729681e.yaml @@ -0,0 +1,7 @@ +--- +desc: Fixed the Axon HTTP upload API starting an upload in the Axon for a request + which was denied. +desc:literal: false +prs: [] +type: bug +... diff --git a/synapse/axon.py b/synapse/axon.py index fc6495057b4..f7c3f545303 100644 --- a/synapse/axon.py +++ b/synapse/axon.py @@ -48,6 +48,7 @@ async def prepare(self): if not await self.allowed(('axon', 'upload')): await self.finish() + return # max_body_size defaults to 100MB and requires a value self.request.connection.set_max_body_size(MAX_HTTP_UPLOAD_SIZE) diff --git a/synapse/cortex.py b/synapse/cortex.py index 71e33dcf77f..fdbf758cf80 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -176,15 +176,18 @@ async def wrap_liftgenr(iden, genr): class CortexAxonMixin: - async def prepare(self): + async def allowed(self, perm, default=False, gateiden=None): + # wait for the Axon only once the user has the permission + if not await super().allowed(perm, default=default, gateiden=gateiden): + return False + try: await self.cell.getAxon() except s_exc.TimeOut as e: self.sendRestExc(e, status_code=http.HTTPStatus.SERVICE_UNAVAILABLE) - await self.finish() - return + return False - await s_coro.ornot(super().prepare) + return True def getAxon(self): return self.cell.axon @@ -199,8 +202,7 @@ class CortexAxonHttpDelV1(CortexAxonMixin, s_axon.AxonHttpDelV1): pass class CortexAxonHttpUploadV1(CortexAxonMixin, s_axon.AxonHttpUploadV1): - # set in prepare(), which is skipped if the Axon is not ready - upfd = None + pass class CortexAxonHttpBySha256V1(CortexAxonMixin, s_axon.AxonHttpBySha256V1): pass diff --git a/synapse/tests/test_axon.py b/synapse/tests/test_axon.py index e5737d8e90e..0906089eddc 100644 --- a/synapse/tests/test_axon.py +++ b/synapse/tests/test_axon.py @@ -612,10 +612,13 @@ async def runAxonTestHttp(self, axon, realaxon=None): item = await resp.json() self.eq('err', item.get('status')) - async with sess.post(url_ul, data=abuf) as resp: - self.eq(resp.status, http.HTTPStatus.FORBIDDEN) - item = await resp.json() - self.eq('err', item.get('status')) + # a denied upload does not start an upload in the Axon + with mock.patch.object(realaxon, 'upload', wraps=realaxon.upload) as upload: + async with sess.post(url_ul, data=abuf) as resp: + self.eq(resp.status, http.HTTPStatus.FORBIDDEN) + item = await resp.json() + self.eq('err', item.get('status')) + upload.assert_not_called() # Stream file byts = io.BytesIO(bbuf) diff --git a/synapse/tests/test_lib_httpapi.py b/synapse/tests/test_lib_httpapi.py index 0cce5fac140..ead0ab7d6ee 100644 --- a/synapse/tests/test_lib_httpapi.py +++ b/synapse/tests/test_lib_httpapi.py @@ -2196,28 +2196,49 @@ async def test_core_remote_axon_http(self): host, port = await core.addHttpsPort(0, host='127.0.0.1') + root = await core.auth.getUserByName('root') + await root.setPasswd('root') + + newb = await core.auth.addUser('newb') + await newb.setPasswd('secret') + + await axon.fini() + + sha256 = s_common.ehex(s_t_axon.asdfhash) + hasurl = f'https://localhost:{port}/api/v1/axon/files/has/sha256/{sha256}' + puturl = f'https://localhost:{port}/api/v1/axon/files/put' + + # auth and permission checks run before waiting on the Axon async with self.getHttpSess() as sess: - await axon.fini() + async with sess.get(hasurl, timeout=timeout) as resp: + self.eq(resp.status, http.HTTPStatus.UNAUTHORIZED) + + async with self.getHttpSess(auth=('newb', 'secret'), port=port) as sess: + + async with sess.get(hasurl, timeout=timeout) as resp: + self.eq(resp.status, http.HTTPStatus.FORBIDDEN) + + async with sess.post(puturl, data=b'asdfasdf', timeout=timeout) as resp: + self.eq(resp.status, http.HTTPStatus.FORBIDDEN) + + async with self.getHttpSess(auth=('root', 'root'), port=port) as sess: with self.raises(TimeoutError): - sha256 = s_common.ehex(s_t_axon.asdfhash) - url = f'https://localhost:{port}/api/v1/axon/files/has/sha256/{sha256}' - async with sess.get(url, timeout=timeout) as resp: + async with sess.get(hasurl, timeout=timeout) as resp: pass with mock.patch.object(core, 'getAxon', functools.partial(core.getAxon, timeout=0.1)): - async with sess.get(url) as resp: + async with sess.get(hasurl) as resp: self.eq(resp.status, http.HTTPStatus.SERVICE_UNAVAILABLE) item = await resp.json() self.eq(item.get('status'), 'err') self.eq(item.get('code'), 'TimeOut') self.eq(item.get('mesg'), 'Timed out waiting 0.1 seconds for the Axon to be ready.') - # the upload handler cleans up without a prepared upload - url = f'https://localhost:{port}/api/v1/axon/files/put' + # the upload handler finishes without starting an upload with self.getLoggerStream('tornado.application') as stream: - async with sess.post(url, data=b'asdfasdf') as resp: + async with sess.post(puturl, data=b'asdfasdf') as resp: self.eq(resp.status, http.HTTPStatus.SERVICE_UNAVAILABLE) item = await resp.json() self.eq(item.get('code'), 'TimeOut') From 10fe832ea8baab6fcb71c29a7beeb1facdc28a06 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 16:39:07 +0000 Subject: [PATCH 12/14] SYN-11748: Apply review fixes - Rename getAxonVersion() to getAxonSynapseVersion(). - Wait on the Axon in $lib.axon.metrics(), Cortex.exportStormToAxon() and Cortex.feedFromAxon() (after its permission check). - Avoid a throwaway dict default in the unpack() feature check. - Drop axon:version from getCellInfo(); telepath getCellInfo() is available to any authenticated user. --- changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml | 5 ++--- docs/synapse/devopsguide.rst | 3 +-- synapse/cortex.py | 13 ++++--------- synapse/lib/stormhttp.py | 2 +- synapse/lib/stormtypes.py | 12 +++++++----- synapse/tests/test_cortex.py | 16 +++++++++++----- synapse/tests/test_lib_stormtypes.py | 12 ++++++++---- 7 files changed, 34 insertions(+), 29 deletions(-) diff --git a/changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml b/changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml index 20f5bf7f433..d16fa4f7ba0 100644 --- a/changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml +++ b/changes/da926d2af70e7dfa59a5eddb8bbace6c.yaml @@ -1,7 +1,6 @@ --- -desc: Added ``axon:ready`` and ``axon:version`` keys to the ``cell`` section of the - Cortex ``getCellInfo()`` API, which report whether the Axon is ready and the Synapse - version most recently reported by the Axon. +desc: Added an ``axon:ready`` key to the ``cell`` section of the Cortex ``getCellInfo()`` + API, which reports whether the Axon is ready. desc:literal: false prs: [] type: feat diff --git a/docs/synapse/devopsguide.rst b/docs/synapse/devopsguide.rst index d3ceebeb7e6..b48bddec7af 100644 --- a/docs/synapse/devopsguide.rst +++ b/docs/synapse/devopsguide.rst @@ -1341,8 +1341,7 @@ Docker Image: ``vertexproject/synapse-axon:v2.x.x`` access by other Advanced Power-Ups. The Cortex reports the status of its Axon in the ``cell`` section of the dictionary returned by ``getCellInfo()``. -The ``axon:ready`` key is ``true`` when the Axon is ready, and the ``axon:version`` key contains the Synapse version -most recently reported by the Axon, or ``null`` if the Cortex has not connected to it. +The ``axon:ready`` key is ``true`` when the Axon is ready. **Configuration** diff --git a/synapse/cortex.py b/synapse/cortex.py index fdbf758cf80..b639b770cad 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -6206,21 +6206,13 @@ async def getCellInfo(self): Notes: In addition to the base Cell information, the ``cell`` section - includes ``axon:ready``, which is True when the Axon is ready, and - ``axon:version``, the Synapse version most recently reported by - the Axon or None if the Cortex has not connected to it. + includes ``axon:ready``, which is True when the Axon is ready. Returns: Dict: A Dictionary of metadata. ''' info = await super().getCellInfo() - - axonvers = None - if (syninfo := self.axoninfo.get('synapse')) is not None: - axonvers = syninfo.get('version') - info['cell']['axon:ready'] = self.axready.is_set() - info['cell']['axon:version'] = axonvers return info def setFeedFunc(self, name, func): @@ -6544,6 +6536,7 @@ async def exportStorm(self, text, opts=None): yield pode async def exportStormToAxon(self, text, opts=None): + await self.getAxon() async with await self.axon.upload() as fd: async for pode in self.exportStorm(text, opts=opts): await fd.write(s_msgpack.en(pode)) @@ -6564,6 +6557,8 @@ async def feedFromAxon(self, sha256, opts=None): # ensure that the user can make all node edits in the layer user.confirm(('node',), gateiden=view.layers[0].iden) + await self.getAxon() + q = s_queue.Queue(maxsize=10000) feedexc = None diff --git a/synapse/lib/stormhttp.py b/synapse/lib/stormhttp.py index cae06351b5e..ac7a31ebc06 100644 --- a/synapse/lib/stormhttp.py +++ b/synapse/lib/stormhttp.py @@ -425,7 +425,7 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, kwargs['proxy'] = proxy if ssl_opts is not None: - axonvers = await s_stormtypes.getAxonVersion(self.runt) + axonvers = await s_stormtypes.getAxonSynapseVersion(self.runt) mesg = f'The ssl_opts argument requires an Axon Synapse version {s_stormtypes.AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, s_stormtypes.AXON_MINVERS_SSLOPTS, mesg=mesg) diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index dfb862cd1f6..9e22a05a82f 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -108,7 +108,7 @@ async def resolveCoreProxyUrl(valu): case _: raise s_exc.BadArg(mesg='HTTP proxy argument must be a string or bool.') -async def getAxonVersion(runt): +async def getAxonSynapseVersion(runt): ''' Get the Synapse version of the Cortex's Axon. @@ -142,7 +142,7 @@ async def resolveAxonProxyArg(valu): ''' runt = s_scope.get('runt') - axonvers = await getAxonVersion(runt) + axonvers = await getAxonSynapseVersion(runt) if axonvers < AXON_MINVERS_PROXY: await runt.snap.warnonce(f'Axon version does not support proxy argument: {axonvers} < {AXON_MINVERS_PROXY}') return False, None @@ -2578,7 +2578,7 @@ async def wget(self, url, headers=None, params=None, method='GET', json=None, bo kwargs['proxy'] = proxy if ssl_opts is not None: - axonvers = await getAxonVersion(self.runt) + axonvers = await getAxonSynapseVersion(self.runt) mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) @@ -2619,7 +2619,7 @@ async def wput(self, sha256, url, headers=None, params=None, method='PUT', kwargs['proxy'] = proxy if ssl_opts is not None: - axonvers = await getAxonVersion(self.runt) + axonvers = await getAxonSynapseVersion(self.runt) mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ f'but the Axon is running {axonvers}' s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) @@ -2728,6 +2728,8 @@ async def csvrows(self, sha256, dialect='excel', errors='ignore', **fmtparams): async def metrics(self): if not self.runt.allowed(('axon', 'has')): self.runt.confirm(('storm', 'lib', 'axon', 'has')) + + await self.runt.snap.core.getAxon() return await self.runt.snap.core.axon.metrics() async def upload(self, genr): @@ -2819,7 +2821,7 @@ async def unpack(self, sha256, fmt, offs=0): await self.runt.snap.core.getAxon() - if self.runt.snap.core.axoninfo.get('features', {}).get('unpack', 0) < 1: + if (features := self.runt.snap.core.axoninfo.get('features')) is None or features.get('unpack', 0) < 1: mesg = 'The connected Axon does not support the the unpack API. Please update your Axon.' raise s_exc.FeatureNotSupported(mesg=mesg) diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index 05f240fe352..56e28ad6f98 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -6870,7 +6870,7 @@ async def test_cortex_axon(self): info = await core.getCellInfo() self.true(info['cell']['axon:ready']) - self.eq(info['cell']['axon:version'], s_version.version) + self.notin('axon:version', info['cell']) self.true(core.axon.isfini) self.false(core.axready.is_set()) @@ -6922,7 +6922,6 @@ async def test_cortex_axon_ready_timeout(self): info = await core.getCellInfo() self.false(info['cell']['axon:ready']) - self.none(info['cell']['axon:version']) sha256 = s_common.ehex(hashlib.sha256(b'vertex').digest()) opts = {'vars': {'sha256': sha256, 'url': 'http://127.0.0.1:1/'}} @@ -6947,6 +6946,8 @@ async def test_cortex_axon_ready_timeout(self): 'return($lib.axon.put($buf))', 'return($lib.axon.hashset($sha256))', 'return($lib.axon.read($sha256))', + 'return($lib.axon.metrics())', + 'return($lib.feed.fromAxon($sha256))', 'return($lib.bytes.put($buf))', 'return($lib.bytes.has($sha256))', 'return($lib.bytes.size($sha256))', @@ -6979,6 +6980,9 @@ async def test_cortex_axon_ready_timeout(self): async for byts in proxy.getAxonBytes(sha256): pass + with self.raises(s_exc.TimeOut): + await proxy.feedFromAxon(sha256) + for query in queries: with self.raises(s_exc.TimeOut): await core.callStorm(query, opts=opts) @@ -6988,6 +6992,10 @@ async def test_cortex_axon_ready_timeout(self): with self.raises(s_exc.AuthDeny): await core.callStorm(query, opts=visiopts) + # an export has no permission of its own to check first + with self.raises(s_exc.TimeOut): + await core.callStorm('return($lib.export.toaxon("inet:fqdn"))') + async with self.getTestAxon(dirn=dirn) as axon: self.true(await s_coro.event_wait(core.axready, timeout=10)) @@ -6995,7 +7003,6 @@ async def test_cortex_axon_ready_timeout(self): info = await core.callStorm('return($lib.cell.getCellInfo())') self.true(info['cell']['axon:ready']) - self.eq(info['cell']['axon:version'], s_version.version) await core.axon.put(b'vertex') @@ -7020,7 +7027,7 @@ async def test_cortex_axon_ready_timeout(self): resp = await core.callStorm(queries[4], opts=opts) self.eq(resp.get('code'), -1) - # the last version reported by the Axon remains after it disconnects + # the Axon is reported as not ready after it disconnects for _ in range(20): if not core.axready.is_set(): break @@ -7028,7 +7035,6 @@ async def test_cortex_axon_ready_timeout(self): info = await core.getCellInfo() self.false(info['cell']['axon:ready']) - self.eq(info['cell']['axon:version'], s_version.version) async def test_cortex_delLayerView(self): diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index a6059f465ac..31a6aee6a44 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -8238,14 +8238,18 @@ async def test_storm_lib_axon_read_unpack(self): visi = await core.auth.addUser('visi') orig_axoninfo = core.axoninfo - core.axoninfo = {'features': {}} data = struct.pack('>Q', 1) size, sha256 = await core.axon.put(data) sha256_s = s_common.ehex(sha256) q = 'return($lib.axon.unpack($sha256, fmt=">Q"))' - await self.asyncraises(s_exc.FeatureNotSupported, - core.callStorm(q, opts={'vars': {'sha256': sha256_s}})) - core.axoninfo = orig_axoninfo + try: + for axoninfo in ({}, {'features': {}}): + core.axoninfo = axoninfo + with self.raises(s_exc.FeatureNotSupported): + await core.callStorm(q, opts={'vars': {'sha256': sha256_s}}) + + finally: + core.axoninfo = orig_axoninfo data = b'vertex.link' size, sha256 = await core.axon.put(data) From 9fb55f2da0b4f99c5e3c41a8bdb4b427788076e5 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 18:12:32 +0000 Subject: [PATCH 13/14] SYN-11748: Catch the Axon wait timeout in getAxon() Wait with s_common.wait_for() and raise TimeOut from its TimeoutError, matching the other timeout handlers. CI coverage did not record the raise that followed a timed out event_wait(). --- synapse/cortex.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/cortex.py b/synapse/cortex.py index b639b770cad..059d76be795 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -6191,9 +6191,12 @@ async def getAxon(self, timeout=s_const.AXON_READY_TIMEOUT): Raises: s_exc.TimeOut: If the Axon is not ready within the timeout. ''' - if not self.axready.is_set() and not await s_coro.event_wait(self.axready, timeout=timeout): - mesg = f'Timed out waiting {timeout} seconds for the Axon to be ready.' - raise s_exc.TimeOut(mesg=mesg, timeout=timeout) + if not self.axready.is_set(): + try: + await s_common.wait_for(self.axready.wait(), timeout) + except asyncio.TimeoutError: + mesg = f'Timed out waiting {timeout} seconds for the Axon to be ready.' + raise s_exc.TimeOut(mesg=mesg, timeout=timeout) from None if (cellinfo := self.axoninfo.get('cell')) is None: return None From 8845a41532626a8f7b5ae2bb23f33122ceb5e753 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 28 Sep 2026 20:13:09 +0000 Subject: [PATCH 14/14] SYN-11748: Tighten the changelog and drop the devops guide change --- changes/c25ec6ffe22011d527e15cec09f6f6be.yaml | 11 +++-------- docs/synapse/devopsguide.rst | 3 --- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml index 3dee80da63a..073861ec547 100644 --- a/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml +++ b/changes/c25ec6ffe22011d527e15cec09f6f6be.yaml @@ -1,12 +1,7 @@ --- -desc: Fixed a ``KeyError`` in the Storm ``$lib.axon.wget()``, ``$lib.axon.wput()``, and - ``$lib.axon.urlfile()`` APIs, and in ``$lib.inet.http`` requests which upload files from - the Axon, when the Axon was not connected. Cortex Storm, Telepath, and HTTP APIs which - use the Axon now wait up to 300 seconds for the Axon to be ready before raising a - ``TimeOut`` error, and Storm APIs raise a ``FeatureNotSupported`` error if the Axon - version is unavailable. The Cortex ``getAxon()`` method now accepts a ``timeout`` - argument, which defaults to 300 seconds, and returns the Axon iden when the Axon is - remote. +desc: Fixed a ``KeyError`` in Storm APIs which interacted with the Axon. Cortex Storm, + Telepath, and HTTP APIs which use the Axon now wait up to 300 seconds for the Axon to be + ready before raising a ``TimeOut`` error. desc:literal: false prs: [] type: bug diff --git a/docs/synapse/devopsguide.rst b/docs/synapse/devopsguide.rst index b48bddec7af..d36fef71b9b 100644 --- a/docs/synapse/devopsguide.rst +++ b/docs/synapse/devopsguide.rst @@ -1340,9 +1340,6 @@ Docker Image: ``vertexproject/synapse-axon:v2.x.x`` it is **highly** recommended that you install it as a separated service to help distribute load and allow direct access by other Advanced Power-Ups. -The Cortex reports the status of its Axon in the ``cell`` section of the dictionary returned by ``getCellInfo()``. -The ``axon:ready`` key is ``true`` when the Axon is ready. - **Configuration** A typical Axon deployment does not require any additional configuration. For the full list supported options, see the